diff --git a/.gitattributes b/.gitattributes index 261276a3d29876775a9f15ab8131ba38ba28ff0d..a089882cf4ceee6b6073b9ebc78904a8831199a9 100644 --- a/.gitattributes +++ b/.gitattributes @@ -45,3 +45,7 @@ f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_2 f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/iter_0002384/__29_1.distcp filter=lfs diff=lfs merge=lfs -text f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/iter_0002384/__21_0.distcp filter=lfs diff=lfs merge=lfs -text f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/iter_0002384/__49_0.distcp filter=lfs diff=lfs merge=lfs -text +f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/iter_0002384/__63_0.distcp filter=lfs diff=lfs merge=lfs -text +f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/iter_0002384/__57_1.distcp filter=lfs diff=lfs merge=lfs -text +f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/iter_0002384/__41_0.distcp filter=lfs diff=lfs merge=lfs -text +f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/iter_0002384/__11_0.distcp filter=lfs diff=lfs merge=lfs -text diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/iter_0002384/__11_0.distcp b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/iter_0002384/__11_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..613ecbf2a200724b368b90f0465127c29035608a --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/iter_0002384/__11_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd7d350b5beb7d34e47a8ac6f2c25abef8f5daa7e97715b19edc86849d913e2f +size 3725391713 diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/iter_0002384/__41_0.distcp b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/iter_0002384/__41_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..90756581fbf514e664d4adc1a2c67db077bfe9bf --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/iter_0002384/__41_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:518273e0fa966a539fe206f84ccd30810060e211e8c0c9d84293b7b9e1738eda +size 3725349062 diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/iter_0002384/__57_1.distcp b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/iter_0002384/__57_1.distcp new file mode 100644 index 0000000000000000000000000000000000000000..29c03ee3ee5e853503c6a90f59ffbe41598ae970 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/iter_0002384/__57_1.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed8e04b5fd632c6346accd2412b82417b3add17587f57b2b8152d07e8ee12c48 +size 3725514944 diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/iter_0002384/__63_0.distcp b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/iter_0002384/__63_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..10c2d0396322cd9c6bc9c5cf54f2723a07ef5392 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/iter_0002384/__63_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ecb70956f1bca0d593382c7b940c03eb20ad1c61f28a154ab2c5ec7c79d91c4b +size 3911098833 diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0000.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0000.json new file mode 100644 index 0000000000000000000000000000000000000000..12ebd76ffda4bd0de6d5139376ae0c67b7c60295 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0000.json @@ -0,0 +1,84 @@ +{ + "rank": 0, + "world_diagnostics_gloo": true, + "aliases": [ + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63 + ], + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7 + ] + ], + "synchronous_singletons": true, + "async_work_handles_preserved": true +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0005.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0005.json new file mode 100644 index 0000000000000000000000000000000000000000..0e48632c1836d4283705a766928ff0fb4e9fcabe --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0005.json @@ -0,0 +1,84 @@ +{ + "rank": 5, + "world_diagnostics_gloo": true, + "aliases": [ + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63 + ], + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7 + ] + ], + "synchronous_singletons": true, + "async_work_handles_preserved": true +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0011.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0011.json new file mode 100644 index 0000000000000000000000000000000000000000..4d9f6fbf0ffd5ee3463f1eb69a1120fc33c7d6bc --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0011.json @@ -0,0 +1,84 @@ +{ + "rank": 11, + "world_diagnostics_gloo": true, + "aliases": [ + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63 + ], + [ + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15 + ] + ], + "synchronous_singletons": true, + "async_work_handles_preserved": true +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0016.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0016.json new file mode 100644 index 0000000000000000000000000000000000000000..1b5cc28483b90df24cda4e3bab76c26ad6294c80 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0016.json @@ -0,0 +1,84 @@ +{ + "rank": 16, + "world_diagnostics_gloo": true, + "aliases": [ + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63 + ], + [ + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23 + ] + ], + "synchronous_singletons": true, + "async_work_handles_preserved": true +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0017.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0017.json new file mode 100644 index 0000000000000000000000000000000000000000..739a42bfe643fa6b7dafff2ed84ae4b6c6f4ebd8 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0017.json @@ -0,0 +1,84 @@ +{ + "rank": 17, + "world_diagnostics_gloo": true, + "aliases": [ + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63 + ], + [ + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23 + ] + ], + "synchronous_singletons": true, + "async_work_handles_preserved": true +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0018.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0018.json new file mode 100644 index 0000000000000000000000000000000000000000..1df8c5b45babd32946cc6a2c996afa66219e2ed5 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0018.json @@ -0,0 +1,84 @@ +{ + "rank": 18, + "world_diagnostics_gloo": true, + "aliases": [ + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63 + ], + [ + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23 + ] + ], + "synchronous_singletons": true, + "async_work_handles_preserved": true +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0019.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0019.json new file mode 100644 index 0000000000000000000000000000000000000000..c1f8a1d7a226ae59f5b4e09af3402385b450d5c3 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0019.json @@ -0,0 +1,84 @@ +{ + "rank": 19, + "world_diagnostics_gloo": true, + "aliases": [ + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63 + ], + [ + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23 + ] + ], + "synchronous_singletons": true, + "async_work_handles_preserved": true +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0021.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0021.json new file mode 100644 index 0000000000000000000000000000000000000000..1c7287bf647ddd96da6bacabff12e08d53ae066e --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0021.json @@ -0,0 +1,84 @@ +{ + "rank": 21, + "world_diagnostics_gloo": true, + "aliases": [ + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63 + ], + [ + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23 + ] + ], + "synchronous_singletons": true, + "async_work_handles_preserved": true +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0022.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0022.json new file mode 100644 index 0000000000000000000000000000000000000000..d9c76cc7d5bb2225bf9082384c56d5e1666afd50 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0022.json @@ -0,0 +1,84 @@ +{ + "rank": 22, + "world_diagnostics_gloo": true, + "aliases": [ + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63 + ], + [ + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23 + ] + ], + "synchronous_singletons": true, + "async_work_handles_preserved": true +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0025.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0025.json new file mode 100644 index 0000000000000000000000000000000000000000..aca618cdee50f95fe05bae069c063d2c5a03f78b --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0025.json @@ -0,0 +1,84 @@ +{ + "rank": 25, + "world_diagnostics_gloo": true, + "aliases": [ + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63 + ], + [ + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + ], + "synchronous_singletons": true, + "async_work_handles_preserved": true +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0029.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0029.json new file mode 100644 index 0000000000000000000000000000000000000000..4a4da2fb82048601affcf3f781f23a3271d6f351 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0029.json @@ -0,0 +1,84 @@ +{ + "rank": 29, + "world_diagnostics_gloo": true, + "aliases": [ + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63 + ], + [ + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + ], + "synchronous_singletons": true, + "async_work_handles_preserved": true +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0031.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0031.json new file mode 100644 index 0000000000000000000000000000000000000000..5f2e8da75e564a233ef79ec898252c3fba6efa02 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0031.json @@ -0,0 +1,84 @@ +{ + "rank": 31, + "world_diagnostics_gloo": true, + "aliases": [ + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63 + ], + [ + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + ], + "synchronous_singletons": true, + "async_work_handles_preserved": true +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0032.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0032.json new file mode 100644 index 0000000000000000000000000000000000000000..2c2f8f691ffaf19100fe91fa458c25a1b818fdf4 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0032.json @@ -0,0 +1,84 @@ +{ + "rank": 32, + "world_diagnostics_gloo": true, + "aliases": [ + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63 + ], + [ + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39 + ] + ], + "synchronous_singletons": true, + "async_work_handles_preserved": true +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0034.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0034.json new file mode 100644 index 0000000000000000000000000000000000000000..207d0c736d2bf432265f0f6bee39cf7be6f4b4c8 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0034.json @@ -0,0 +1,84 @@ +{ + "rank": 34, + "world_diagnostics_gloo": true, + "aliases": [ + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63 + ], + [ + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39 + ] + ], + "synchronous_singletons": true, + "async_work_handles_preserved": true +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0044.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0044.json new file mode 100644 index 0000000000000000000000000000000000000000..9a6681695c36ed8ac6d73f0d09e58cb98372c0c7 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0044.json @@ -0,0 +1,84 @@ +{ + "rank": 44, + "world_diagnostics_gloo": true, + "aliases": [ + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63 + ], + [ + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47 + ] + ], + "synchronous_singletons": true, + "async_work_handles_preserved": true +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0046.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0046.json new file mode 100644 index 0000000000000000000000000000000000000000..444898f030480971437018c30fb0718e8c883ccf --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0046.json @@ -0,0 +1,84 @@ +{ + "rank": 46, + "world_diagnostics_gloo": true, + "aliases": [ + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63 + ], + [ + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47 + ] + ], + "synchronous_singletons": true, + "async_work_handles_preserved": true +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0047.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0047.json new file mode 100644 index 0000000000000000000000000000000000000000..8d96ad2b2cda6f8b914e0c9cca3aa39d13f16962 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0047.json @@ -0,0 +1,84 @@ +{ + "rank": 47, + "world_diagnostics_gloo": true, + "aliases": [ + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63 + ], + [ + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47 + ] + ], + "synchronous_singletons": true, + "async_work_handles_preserved": true +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0048.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0048.json new file mode 100644 index 0000000000000000000000000000000000000000..b536a57826a678cb5583c26e25b34013f5c4b6f9 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0048.json @@ -0,0 +1,84 @@ +{ + "rank": 48, + "world_diagnostics_gloo": true, + "aliases": [ + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63 + ], + [ + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55 + ] + ], + "synchronous_singletons": true, + "async_work_handles_preserved": true +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0053.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0053.json new file mode 100644 index 0000000000000000000000000000000000000000..e3e4b4fecd050122960e88f6e49a8705df70c9e3 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0053.json @@ -0,0 +1,84 @@ +{ + "rank": 53, + "world_diagnostics_gloo": true, + "aliases": [ + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63 + ], + [ + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55 + ] + ], + "synchronous_singletons": true, + "async_work_handles_preserved": true +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0054.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0054.json new file mode 100644 index 0000000000000000000000000000000000000000..c36063cacd054c67a48c790426748c4dcba9b1b2 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0054.json @@ -0,0 +1,84 @@ +{ + "rank": 54, + "world_diagnostics_gloo": true, + "aliases": [ + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63 + ], + [ + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55 + ] + ], + "synchronous_singletons": true, + "async_work_handles_preserved": true +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0056.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0056.json new file mode 100644 index 0000000000000000000000000000000000000000..35baf84455e064c3cd0aea4e58a37c2083cb0be8 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0056.json @@ -0,0 +1,84 @@ +{ + "rank": 56, + "world_diagnostics_gloo": true, + "aliases": [ + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63 + ], + [ + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63 + ] + ], + "synchronous_singletons": true, + "async_work_handles_preserved": true +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0057.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0057.json new file mode 100644 index 0000000000000000000000000000000000000000..465124c3d95494f45b910868c116953a52bdd2ec --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0057.json @@ -0,0 +1,84 @@ +{ + "rank": 57, + "world_diagnostics_gloo": true, + "aliases": [ + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63 + ], + [ + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63 + ] + ], + "synchronous_singletons": true, + "async_work_handles_preserved": true +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0060.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0060.json new file mode 100644 index 0000000000000000000000000000000000000000..a87aeaf0693b68fec85329dcb122c0e4fbb42660 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0060.json @@ -0,0 +1,84 @@ +{ + "rank": 60, + "world_diagnostics_gloo": true, + "aliases": [ + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63 + ], + [ + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63 + ] + ], + "synchronous_singletons": true, + "async_work_handles_preserved": true +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0062.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0062.json new file mode 100644 index 0000000000000000000000000000000000000000..67368a01e8ebcaabe358ba7f578c6704711f73b6 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0062.json @@ -0,0 +1,84 @@ +{ + "rank": 62, + "world_diagnostics_gloo": true, + "aliases": [ + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63 + ], + [ + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63 + ] + ], + "synchronous_singletons": true, + "async_work_handles_preserved": true +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0001.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0001.json new file mode 100644 index 0000000000000000000000000000000000000000..105611ede9e3e990e876eba02e10c72c93fef6af --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0001.json @@ -0,0 +1,13227 @@ +{ + "rank": 1, + "policy": "dc_adamh_named_matrices_v1", + "parameter_groups": [ + { + "name": "module.module.embedding.word_embeddings.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "embedding", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.output_layer.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "lm_head", + "expert": false + } + ], + "family_counts": { + "embedding": 1, + "in_proj": 294, + "out_proj": 49, + "linear_fc1": 56, + "linear_fc2": 56, + "linear_proj": 7, + "linear_qkv": 7, + "lm_head": 1 + }, + "moment_reset": false, + "loaded_step_values": { + "0": [ + 52788.0 + ], + "1": [] + }, + "native_optimizer_types": [ + "FusedAdam", + "FusedAdam" + ], + "full_matrix_norms_across_dp_shards": true, + "tp": 1, + "pp": 1 +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0004.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0004.json new file mode 100644 index 0000000000000000000000000000000000000000..4f37dc56f897e64ffc8e05c0ec13844ca586281e --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0004.json @@ -0,0 +1,13227 @@ +{ + "rank": 4, + "policy": "dc_adamh_named_matrices_v1", + "parameter_groups": [ + { + "name": "module.module.embedding.word_embeddings.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "embedding", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.output_layer.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "lm_head", + "expert": false + } + ], + "family_counts": { + "embedding": 1, + "in_proj": 294, + "out_proj": 49, + "linear_fc1": 56, + "linear_fc2": 56, + "linear_proj": 7, + "linear_qkv": 7, + "lm_head": 1 + }, + "moment_reset": false, + "loaded_step_values": { + "0": [ + 52788.0 + ], + "1": [] + }, + "native_optimizer_types": [ + "FusedAdam", + "FusedAdam" + ], + "full_matrix_norms_across_dp_shards": true, + "tp": 1, + "pp": 1 +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0012.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0012.json new file mode 100644 index 0000000000000000000000000000000000000000..b497729a7271100da29e880ce586496d30ec8bbe --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0012.json @@ -0,0 +1,13227 @@ +{ + "rank": 12, + "policy": "dc_adamh_named_matrices_v1", + "parameter_groups": [ + { + "name": "module.module.embedding.word_embeddings.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "embedding", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.output_layer.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "lm_head", + "expert": false + } + ], + "family_counts": { + "embedding": 1, + "in_proj": 294, + "out_proj": 49, + "linear_fc1": 56, + "linear_fc2": 56, + "linear_proj": 7, + "linear_qkv": 7, + "lm_head": 1 + }, + "moment_reset": false, + "loaded_step_values": { + "0": [ + 52788.0 + ], + "1": [] + }, + "native_optimizer_types": [ + "FusedAdam", + "FusedAdam" + ], + "full_matrix_norms_across_dp_shards": true, + "tp": 1, + "pp": 1 +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0013.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0013.json new file mode 100644 index 0000000000000000000000000000000000000000..21cae2ba1d62b043526c8717963ac1d75d94ad4e --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0013.json @@ -0,0 +1,13227 @@ +{ + "rank": 13, + "policy": "dc_adamh_named_matrices_v1", + "parameter_groups": [ + { + "name": "module.module.embedding.word_embeddings.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "embedding", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.output_layer.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "lm_head", + "expert": false + } + ], + "family_counts": { + "embedding": 1, + "in_proj": 294, + "out_proj": 49, + "linear_fc1": 56, + "linear_fc2": 56, + "linear_proj": 7, + "linear_qkv": 7, + "lm_head": 1 + }, + "moment_reset": false, + "loaded_step_values": { + "0": [ + 52788.0 + ], + "1": [] + }, + "native_optimizer_types": [ + "FusedAdam", + "FusedAdam" + ], + "full_matrix_norms_across_dp_shards": true, + "tp": 1, + "pp": 1 +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0015.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0015.json new file mode 100644 index 0000000000000000000000000000000000000000..8bbf2280c8fb580b20bc9549dc4fa200a25773b4 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0015.json @@ -0,0 +1,13227 @@ +{ + "rank": 15, + "policy": "dc_adamh_named_matrices_v1", + "parameter_groups": [ + { + "name": "module.module.embedding.word_embeddings.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "embedding", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.output_layer.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "lm_head", + "expert": false + } + ], + "family_counts": { + "embedding": 1, + "in_proj": 294, + "out_proj": 49, + "linear_fc1": 56, + "linear_fc2": 56, + "linear_proj": 7, + "linear_qkv": 7, + "lm_head": 1 + }, + "moment_reset": false, + "loaded_step_values": { + "0": [ + 52788.0 + ], + "1": [] + }, + "native_optimizer_types": [ + "FusedAdam", + "FusedAdam" + ], + "full_matrix_norms_across_dp_shards": true, + "tp": 1, + "pp": 1 +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0017.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0017.json new file mode 100644 index 0000000000000000000000000000000000000000..1be6baf95cd6fd277937c059b381ca7be90e6c1c --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0017.json @@ -0,0 +1,13227 @@ +{ + "rank": 17, + "policy": "dc_adamh_named_matrices_v1", + "parameter_groups": [ + { + "name": "module.module.embedding.word_embeddings.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "embedding", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.output_layer.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "lm_head", + "expert": false + } + ], + "family_counts": { + "embedding": 1, + "in_proj": 294, + "out_proj": 49, + "linear_fc1": 56, + "linear_fc2": 56, + "linear_proj": 7, + "linear_qkv": 7, + "lm_head": 1 + }, + "moment_reset": false, + "loaded_step_values": { + "0": [ + 52788.0 + ], + "1": [] + }, + "native_optimizer_types": [ + "FusedAdam", + "FusedAdam" + ], + "full_matrix_norms_across_dp_shards": true, + "tp": 1, + "pp": 1 +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0020.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0020.json new file mode 100644 index 0000000000000000000000000000000000000000..7eb569003d58a0fa7bfe2c412aab735b0feeb07b --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0020.json @@ -0,0 +1,13227 @@ +{ + "rank": 20, + "policy": "dc_adamh_named_matrices_v1", + "parameter_groups": [ + { + "name": "module.module.embedding.word_embeddings.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "embedding", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.output_layer.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "lm_head", + "expert": false + } + ], + "family_counts": { + "embedding": 1, + "in_proj": 294, + "out_proj": 49, + "linear_fc1": 56, + "linear_fc2": 56, + "linear_proj": 7, + "linear_qkv": 7, + "lm_head": 1 + }, + "moment_reset": false, + "loaded_step_values": { + "0": [ + 52788.0 + ], + "1": [] + }, + "native_optimizer_types": [ + "FusedAdam", + "FusedAdam" + ], + "full_matrix_norms_across_dp_shards": true, + "tp": 1, + "pp": 1 +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0025.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0025.json new file mode 100644 index 0000000000000000000000000000000000000000..8165645ef7ac728cf504c38e694bcca379ba6c06 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0025.json @@ -0,0 +1,13227 @@ +{ + "rank": 25, + "policy": "dc_adamh_named_matrices_v1", + "parameter_groups": [ + { + "name": "module.module.embedding.word_embeddings.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "embedding", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.output_layer.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "lm_head", + "expert": false + } + ], + "family_counts": { + "embedding": 1, + "in_proj": 294, + "out_proj": 49, + "linear_fc1": 56, + "linear_fc2": 56, + "linear_proj": 7, + "linear_qkv": 7, + "lm_head": 1 + }, + "moment_reset": false, + "loaded_step_values": { + "0": [ + 52788.0 + ], + "1": [] + }, + "native_optimizer_types": [ + "FusedAdam", + "FusedAdam" + ], + "full_matrix_norms_across_dp_shards": true, + "tp": 1, + "pp": 1 +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0026.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0026.json new file mode 100644 index 0000000000000000000000000000000000000000..a6158161665f7f736f4aeb8774efd546a910052a --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0026.json @@ -0,0 +1,13227 @@ +{ + "rank": 26, + "policy": "dc_adamh_named_matrices_v1", + "parameter_groups": [ + { + "name": "module.module.embedding.word_embeddings.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "embedding", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.output_layer.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "lm_head", + "expert": false + } + ], + "family_counts": { + "embedding": 1, + "in_proj": 294, + "out_proj": 49, + "linear_fc1": 56, + "linear_fc2": 56, + "linear_proj": 7, + "linear_qkv": 7, + "lm_head": 1 + }, + "moment_reset": false, + "loaded_step_values": { + "0": [ + 52788.0 + ], + "1": [] + }, + "native_optimizer_types": [ + "FusedAdam", + "FusedAdam" + ], + "full_matrix_norms_across_dp_shards": true, + "tp": 1, + "pp": 1 +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0031.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0031.json new file mode 100644 index 0000000000000000000000000000000000000000..6dc6236682f7f693be9afebd1b42db46488a34bd --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0031.json @@ -0,0 +1,13227 @@ +{ + "rank": 31, + "policy": "dc_adamh_named_matrices_v1", + "parameter_groups": [ + { + "name": "module.module.embedding.word_embeddings.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "embedding", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.output_layer.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "lm_head", + "expert": false + } + ], + "family_counts": { + "embedding": 1, + "in_proj": 294, + "out_proj": 49, + "linear_fc1": 56, + "linear_fc2": 56, + "linear_proj": 7, + "linear_qkv": 7, + "lm_head": 1 + }, + "moment_reset": false, + "loaded_step_values": { + "0": [ + 52788.0 + ], + "1": [] + }, + "native_optimizer_types": [ + "FusedAdam", + "FusedAdam" + ], + "full_matrix_norms_across_dp_shards": true, + "tp": 1, + "pp": 1 +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0032.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0032.json new file mode 100644 index 0000000000000000000000000000000000000000..2172f2ae67d08bb801d31380b82a98e0318587e3 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0032.json @@ -0,0 +1,13227 @@ +{ + "rank": 32, + "policy": "dc_adamh_named_matrices_v1", + "parameter_groups": [ + { + "name": "module.module.embedding.word_embeddings.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "embedding", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.output_layer.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "lm_head", + "expert": false + } + ], + "family_counts": { + "embedding": 1, + "in_proj": 294, + "out_proj": 49, + "linear_fc1": 56, + "linear_fc2": 56, + "linear_proj": 7, + "linear_qkv": 7, + "lm_head": 1 + }, + "moment_reset": false, + "loaded_step_values": { + "0": [ + 52788.0 + ], + "1": [] + }, + "native_optimizer_types": [ + "FusedAdam", + "FusedAdam" + ], + "full_matrix_norms_across_dp_shards": true, + "tp": 1, + "pp": 1 +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0034.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0034.json new file mode 100644 index 0000000000000000000000000000000000000000..1cba266af2a47b842b6f9ea0b0fe826ac4f08b28 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0034.json @@ -0,0 +1,13227 @@ +{ + "rank": 34, + "policy": "dc_adamh_named_matrices_v1", + "parameter_groups": [ + { + "name": "module.module.embedding.word_embeddings.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "embedding", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.output_layer.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "lm_head", + "expert": false + } + ], + "family_counts": { + "embedding": 1, + "in_proj": 294, + "out_proj": 49, + "linear_fc1": 56, + "linear_fc2": 56, + "linear_proj": 7, + "linear_qkv": 7, + "lm_head": 1 + }, + "moment_reset": false, + "loaded_step_values": { + "0": [ + 52788.0 + ], + "1": [] + }, + "native_optimizer_types": [ + "FusedAdam", + "FusedAdam" + ], + "full_matrix_norms_across_dp_shards": true, + "tp": 1, + "pp": 1 +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0037.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0037.json new file mode 100644 index 0000000000000000000000000000000000000000..e58fa47ccc039e9be03446dda4bd3ba07eb099a3 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0037.json @@ -0,0 +1,13227 @@ +{ + "rank": 37, + "policy": "dc_adamh_named_matrices_v1", + "parameter_groups": [ + { + "name": "module.module.embedding.word_embeddings.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "embedding", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.output_layer.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "lm_head", + "expert": false + } + ], + "family_counts": { + "embedding": 1, + "in_proj": 294, + "out_proj": 49, + "linear_fc1": 56, + "linear_fc2": 56, + "linear_proj": 7, + "linear_qkv": 7, + "lm_head": 1 + }, + "moment_reset": false, + "loaded_step_values": { + "0": [ + 52788.0 + ], + "1": [] + }, + "native_optimizer_types": [ + "FusedAdam", + "FusedAdam" + ], + "full_matrix_norms_across_dp_shards": true, + "tp": 1, + "pp": 1 +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0038.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0038.json new file mode 100644 index 0000000000000000000000000000000000000000..f486f1e0baae21541e11a283a83749d8de293ffa --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0038.json @@ -0,0 +1,13227 @@ +{ + "rank": 38, + "policy": "dc_adamh_named_matrices_v1", + "parameter_groups": [ + { + "name": "module.module.embedding.word_embeddings.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "embedding", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.output_layer.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "lm_head", + "expert": false + } + ], + "family_counts": { + "embedding": 1, + "in_proj": 294, + "out_proj": 49, + "linear_fc1": 56, + "linear_fc2": 56, + "linear_proj": 7, + "linear_qkv": 7, + "lm_head": 1 + }, + "moment_reset": false, + "loaded_step_values": { + "0": [ + 52788.0 + ], + "1": [] + }, + "native_optimizer_types": [ + "FusedAdam", + "FusedAdam" + ], + "full_matrix_norms_across_dp_shards": true, + "tp": 1, + "pp": 1 +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0041.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0041.json new file mode 100644 index 0000000000000000000000000000000000000000..c7807e09ff7850fba1bb73f174a420e4db30cf6b --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0041.json @@ -0,0 +1,13227 @@ +{ + "rank": 41, + "policy": "dc_adamh_named_matrices_v1", + "parameter_groups": [ + { + "name": "module.module.embedding.word_embeddings.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "embedding", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.output_layer.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "lm_head", + "expert": false + } + ], + "family_counts": { + "embedding": 1, + "in_proj": 294, + "out_proj": 49, + "linear_fc1": 56, + "linear_fc2": 56, + "linear_proj": 7, + "linear_qkv": 7, + "lm_head": 1 + }, + "moment_reset": false, + "loaded_step_values": { + "0": [ + 52788.0 + ], + "1": [] + }, + "native_optimizer_types": [ + "FusedAdam", + "FusedAdam" + ], + "full_matrix_norms_across_dp_shards": true, + "tp": 1, + "pp": 1 +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0042.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0042.json new file mode 100644 index 0000000000000000000000000000000000000000..45cededa57e93ea2981fc879a3b02feb4931c645 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0042.json @@ -0,0 +1,13227 @@ +{ + "rank": 42, + "policy": "dc_adamh_named_matrices_v1", + "parameter_groups": [ + { + "name": "module.module.embedding.word_embeddings.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "embedding", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.output_layer.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "lm_head", + "expert": false + } + ], + "family_counts": { + "embedding": 1, + "in_proj": 294, + "out_proj": 49, + "linear_fc1": 56, + "linear_fc2": 56, + "linear_proj": 7, + "linear_qkv": 7, + "lm_head": 1 + }, + "moment_reset": false, + "loaded_step_values": { + "0": [ + 52788.0 + ], + "1": [] + }, + "native_optimizer_types": [ + "FusedAdam", + "FusedAdam" + ], + "full_matrix_norms_across_dp_shards": true, + "tp": 1, + "pp": 1 +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0043.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0043.json new file mode 100644 index 0000000000000000000000000000000000000000..25cf0eeb8f2702d57348c63d5cc9d5a1f7684780 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0043.json @@ -0,0 +1,13227 @@ +{ + "rank": 43, + "policy": "dc_adamh_named_matrices_v1", + "parameter_groups": [ + { + "name": "module.module.embedding.word_embeddings.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "embedding", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.output_layer.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "lm_head", + "expert": false + } + ], + "family_counts": { + "embedding": 1, + "in_proj": 294, + "out_proj": 49, + "linear_fc1": 56, + "linear_fc2": 56, + "linear_proj": 7, + "linear_qkv": 7, + "lm_head": 1 + }, + "moment_reset": false, + "loaded_step_values": { + "0": [ + 52788.0 + ], + "1": [] + }, + "native_optimizer_types": [ + "FusedAdam", + "FusedAdam" + ], + "full_matrix_norms_across_dp_shards": true, + "tp": 1, + "pp": 1 +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0049.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0049.json new file mode 100644 index 0000000000000000000000000000000000000000..92b98f523d18663ab7d3b7604ab10cf535b6a450 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0049.json @@ -0,0 +1,13227 @@ +{ + "rank": 49, + "policy": "dc_adamh_named_matrices_v1", + "parameter_groups": [ + { + "name": "module.module.embedding.word_embeddings.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "embedding", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.output_layer.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "lm_head", + "expert": false + } + ], + "family_counts": { + "embedding": 1, + "in_proj": 294, + "out_proj": 49, + "linear_fc1": 56, + "linear_fc2": 56, + "linear_proj": 7, + "linear_qkv": 7, + "lm_head": 1 + }, + "moment_reset": false, + "loaded_step_values": { + "0": [ + 52788.0 + ], + "1": [] + }, + "native_optimizer_types": [ + "FusedAdam", + "FusedAdam" + ], + "full_matrix_norms_across_dp_shards": true, + "tp": 1, + "pp": 1 +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0050.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0050.json new file mode 100644 index 0000000000000000000000000000000000000000..92b1b859f2acb31db41b6cb662215162549f1838 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0050.json @@ -0,0 +1,13227 @@ +{ + "rank": 50, + "policy": "dc_adamh_named_matrices_v1", + "parameter_groups": [ + { + "name": "module.module.embedding.word_embeddings.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "embedding", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.output_layer.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "lm_head", + "expert": false + } + ], + "family_counts": { + "embedding": 1, + "in_proj": 294, + "out_proj": 49, + "linear_fc1": 56, + "linear_fc2": 56, + "linear_proj": 7, + "linear_qkv": 7, + "lm_head": 1 + }, + "moment_reset": false, + "loaded_step_values": { + "0": [ + 52788.0 + ], + "1": [] + }, + "native_optimizer_types": [ + "FusedAdam", + "FusedAdam" + ], + "full_matrix_norms_across_dp_shards": true, + "tp": 1, + "pp": 1 +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0051.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0051.json new file mode 100644 index 0000000000000000000000000000000000000000..11cac7a9a64c95e1752a99187af346be5478d689 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0051.json @@ -0,0 +1,13227 @@ +{ + "rank": 51, + "policy": "dc_adamh_named_matrices_v1", + "parameter_groups": [ + { + "name": "module.module.embedding.word_embeddings.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "embedding", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.output_layer.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "lm_head", + "expert": false + } + ], + "family_counts": { + "embedding": 1, + "in_proj": 294, + "out_proj": 49, + "linear_fc1": 56, + "linear_fc2": 56, + "linear_proj": 7, + "linear_qkv": 7, + "lm_head": 1 + }, + "moment_reset": false, + "loaded_step_values": { + "0": [ + 52788.0 + ], + "1": [] + }, + "native_optimizer_types": [ + "FusedAdam", + "FusedAdam" + ], + "full_matrix_norms_across_dp_shards": true, + "tp": 1, + "pp": 1 +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb/wandb/run-20261011_160551-acr8mx9p/files/output.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb/wandb/run-20261011_160551-acr8mx9p/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..b45d0e74e7421b600248a86b5df537332a453be8 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb/wandb/run-20261011_160551-acr8mx9p/files/output.log @@ -0,0 +1,14281 @@ +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +times across ranks (ms): + load-checkpoint: + rank 0: 107363.73 + rank 1: 107363.66 + rank 2: 107363.73 + rank 3: 107363.94 + rank 4: 107363.67 + rank 5: 107363.82 + rank 6: 107363.65 + rank 7: 107363.67 + rank 8: 107363.52 + rank 9: 107363.52 + rank 10: 107363.68 + rank 11: 107363.62 + rank 12: 107363.53 + rank 13: 107363.49 + rank 14: 107363.53 + rank 15: 107363.47 + rank 16: 107363.80 + rank 17: 107363.73 + rank 18: 107363.80 + rank 19: 107363.84 + rank 20: 107363.67 + rank 21: 107363.92 + rank 22: 107364.05 + rank 23: 107363.88 + rank 24: 107364.27 + rank 25: 107364.05 + rank 26: 107364.22 + rank 27: 107364.56 + rank 28: 107364.23 + rank 29: 107364.26 + rank 30: 107364.14 + rank 31: 107364.10 + rank 32: 107363.66 + rank 33: 107363.95 + rank 34: 107363.64 + rank 35: 107363.93 + rank 36: 107364.06 + rank 37: 107363.80 + rank 38: 107363.63 + rank 39: 107363.64 + rank 40: 107363.84 + rank 41: 107363.89 + rank 42: 107363.10 + rank 43: 107363.90 + rank 44: 107363.88 + rank 45: 107363.77 + rank 46: 107363.84 + rank 47: 107363.82 + rank 48: 107363.55 + rank 49: 107363.66 + rank 50: 107363.49 + rank 51: 107363.55 + rank 52: 107363.55 + rank 53: 107363.81 + rank 54: 107363.53 + rank 55: 107363.48 + rank 56: 107364.07 + rank 57: 107363.88 + rank 58: 107363.97 + rank 59: 107363.88 + rank 60: 107363.92 + rank 61: 107363.88 + rank 62: 107363.87 + rank 63: 107363.84 +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v +>>> start_samples: 0 +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v +pad_document_marker: 12051293 +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B 3901db7a83bb2b1721a94d6b726d388f-GPTDataset-train-obfd cache hit: True +>>> start_samples: 0 +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v3_part1_40B_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_16/stage_16_final_20261010_v +/mnt/hdd/cpt/binidx_cache_stage16_final_20261010_v3_part1_40B fb7694ad3d3e6e6ec6a516dc8a6965ee-GPTDataset-valid-random cache hit: True +times across ranks (ms): + model-and-optimizer-setup: + rank 0: 133594.25 + rank 1: 133588.12 + rank 2: 133585.66 + rank 3: 133592.48 + rank 4: 133588.34 + rank 5: 133591.12 + rank 6: 133591.62 + rank 7: 133587.55 + rank 8: 133585.38 + rank 9: 133585.20 + rank 10: 133586.12 + rank 11: 133585.22 + rank 12: 133588.14 + rank 13: 133587.11 + rank 14: 133589.19 + rank 15: 133590.19 + rank 16: 133591.02 + rank 17: 133592.94 + rank 18: 133593.62 + rank 19: 133592.02 + rank 20: 133593.56 + rank 21: 133594.33 + rank 22: 133594.67 + rank 23: 133595.20 + rank 24: 133599.69 + rank 25: 133605.33 + rank 26: 133601.14 + rank 27: 133601.34 + rank 28: 133614.59 + rank 29: 133599.73 + rank 30: 133598.86 + rank 31: 133606.92 + rank 32: 133594.98 + rank 33: 133598.09 + rank 34: 133598.23 + rank 35: 133595.41 + rank 36: 133597.55 + rank 37: 133597.16 + rank 38: 133594.70 + rank 39: 133596.27 + rank 40: 133595.14 + rank 41: 133592.97 + rank 42: 133594.67 + rank 43: 133593.94 + rank 44: 133591.88 + rank 45: 133591.97 + rank 46: 133590.47 + rank 47: 133593.95 + rank 48: 133592.89 + rank 49: 133590.12 + rank 50: 133593.62 + rank 51: 133593.09 + rank 52: 133594.17 + rank 53: 133589.03 + rank 54: 133591.39 + rank 55: 133592.58 + rank 56: 133586.06 + rank 57: 133587.73 + rank 58: 133585.34 + rank 59: 133585.81 + rank 60: 133586.86 + rank 61: 133585.80 + rank 62: 133585.98 + rank 63: 133585.05 + train/valid/test-data-iterators-setup: + rank 0: 22041.88 + rank 1: 25482.36 + rank 2: 25482.49 + rank 3: 25381.98 + rank 4: 25382.04 + rank 5: 25371.71 + rank 6: 25371.76 + rank 7: 25371.91 + rank 8: 25482.21 + rank 9: 25482.11 + rank 10: 25482.09 + rank 11: 25482.12 + rank 12: 25482.31 + rank 13: 25482.14 + rank 14: 25482.14 + rank 15: 25482.26 + rank 16: 25482.48 + rank 17: 25495.04 + rank 18: 25495.08 + rank 19: 25495.08 + rank 20: 25482.49 + rank 21: 25482.28 + rank 22: 25482.37 + rank 23: 25482.21 + rank 24: 26596.63 + rank 25: 26649.74 + rank 26: 26649.72 + rank 27: 26649.69 + rank 28: 26649.92 + rank 29: 26649.71 + rank 30: 26596.54 + rank 31: 26605.73 + rank 32: 26649.84 + rank 33: 26649.83 + rank 34: 26650.61 + rank 35: 26649.81 + rank 36: 26649.79 + rank 37: 26649.81 + rank 38: 26649.95 + rank 39: 26649.79 + rank 40: 26649.96 + rank 41: 26649.87 + rank 42: 26650.04 + rank 43: 26649.88 + rank 44: 26649.96 + rank 45: 26649.68 + rank 46: 26649.87 + rank 47: 26649.71 + rank 48: 26649.87 + rank 49: 26649.86 + rank 50: 26649.82 + rank 51: 26649.88 + rank 52: 26649.72 + rank 53: 26650.77 + rank 54: 26649.83 + rank 55: 26649.85 + rank 56: 26650.07 + rank 57: 26649.82 + rank 58: 26649.92 + rank 59: 26649.81 + rank 60: 26650.04 + rank 61: 26650.07 + rank 62: 26649.93 + rank 63: 26650.07 +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. + [2026-10-11 16:20:03] iteration 41/ 2384 | consumed samples: 1312 | elapsed time per iteration (ms): 326507.2 | throughput per GPU (TFLOP/s/GPU): 15.5 | MFU 4.97% | FLOPs estimator: workload-aware | logical tokens/s: 51383.9 | valid tokens/s: 51383.9 | physical tokens/s: 51643.0 | loss tokens/s: 40349.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.411327E-01 | lm-global-supervised-tokens: 1.317437E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.743055E+01 | lm-token-buffer-mib: 2.323268E+02 | global_load_balancing_loss: 1.000853E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11233.623 | params rms: 0.060826 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8 days, 20:30:06.479095 | finish at 2026-10-20 12:50:09 + [2026-10-11 16:23:48] iteration 42/ 2384 | consumed samples: 1344 | elapsed time per iteration (ms): 225493.8 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.58% | FLOPs estimator: workload-aware | logical tokens/s: 74401.9 | valid tokens/s: 74401.9 | physical tokens/s: 75730.1 | loss tokens/s: 51104.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.369315E-01 | lm-global-supervised-tokens: 1.152365E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.250638E-02 | lm-token-buffer-mib: 2.364373E+02 | global_load_balancing_loss: 9.989473E-01 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11233.552 | params rms: 0.060826 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 2:41:46.593333 | finish at 2026-10-17 19:05:35 + [2026-10-11 16:27:32] iteration 43/ 2384 | consumed samples: 1376 | elapsed time per iteration (ms): 223354.2 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.71% | FLOPs estimator: workload-aware | logical tokens/s: 75114.7 | valid tokens/s: 75114.7 | physical tokens/s: 75546.9 | loss tokens/s: 56547.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.432067E-01 | lm-global-supervised-tokens: 1.263020E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.027606E-02 | lm-token-buffer-mib: 2.333867E+02 | global_load_balancing_loss: 1.001140E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11233.481 | params rms: 0.060826 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:14:32.246120 | finish at 2026-10-17 17:42:04 + [2026-10-11 16:31:12] iteration 44/ 2384 | consumed samples: 1408 | elapsed time per iteration (ms): 220272.1 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.93% | FLOPs estimator: workload-aware | logical tokens/s: 76165.7 | valid tokens/s: 76165.7 | physical tokens/s: 76784.7 | loss tokens/s: 56930.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.636720E-01 | lm-global-supervised-tokens: 1.254026E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.429243E-02 | lm-token-buffer-mib: 2.347867E+02 | global_load_balancing_loss: 1.000705E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11233.414 | params rms: 0.060825 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:10:36.797061 | finish at 2026-10-17 15:41:49 + [2026-10-11 16:35:50] iteration 45/ 2384 | consumed samples: 1440 | elapsed time per iteration (ms): 277810.0 | throughput per GPU (TFLOP/s/GPU): 17.4 | MFU 5.58% | FLOPs estimator: workload-aware | logical tokens/s: 60391.0 | valid tokens/s: 60391.0 | physical tokens/s: 62055.2 | loss tokens/s: 45216.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.028783E-01 | lm-global-supervised-tokens: 1.256153E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.588596E-02 | lm-token-buffer-mib: 2.347526E+02 | global_load_balancing_loss: 1.001366E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11233.347 | params rms: 0.060825 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 12:29:57.637825 | finish at 2026-10-19 05:05:47 + [2026-10-11 16:39:32] iteration 46/ 2384 | consumed samples: 1472 | elapsed time per iteration (ms): 222489.3 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.47% | FLOPs estimator: workload-aware | logical tokens/s: 75406.8 | valid tokens/s: 75406.8 | physical tokens/s: 76199.1 | loss tokens/s: 60014.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.189202E-01 | lm-global-supervised-tokens: 1.335262E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.551362E-02 | lm-token-buffer-mib: 2.342603E+02 | global_load_balancing_loss: 1.003794E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11233.280 | params rms: 0.060825 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 0:29:39.874175 | finish at 2026-10-17 17:09:12 + [2026-10-11 16:43:13] iteration 47/ 2384 | consumed samples: 1504 | elapsed time per iteration (ms): 220679.1 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.15% | FLOPs estimator: workload-aware | logical tokens/s: 76025.2 | valid tokens/s: 76025.2 | physical tokens/s: 76465.4 | loss tokens/s: 60131.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.462784E-01 | lm-global-supervised-tokens: 1.326972E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.748896E-02 | lm-token-buffer-mib: 2.341247E+02 | global_load_balancing_loss: 1.001103E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11233.213 | params rms: 0.060824 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:15:27.030598 | finish at 2026-10-17 15:58:40 + [2026-10-11 16:46:53] iteration 48/ 2384 | consumed samples: 1536 | elapsed time per iteration (ms): 220024.6 | throughput per GPU (TFLOP/s/GPU): 23.2 | MFU 7.43% | FLOPs estimator: workload-aware | logical tokens/s: 76251.3 | valid tokens/s: 76251.3 | physical tokens/s: 76638.1 | loss tokens/s: 57205.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.446347E-01 | lm-global-supervised-tokens: 1.258657E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.075306E-02 | lm-token-buffer-mib: 2.324507E+02 | global_load_balancing_loss: 9.990532E-01 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11233.148 | params rms: 0.060824 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:46:17.414429 | finish at 2026-10-17 15:33:10 + [2026-10-11 16:50:36] iteration 49/ 2384 | consumed samples: 1568 | elapsed time per iteration (ms): 223582.4 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.52% | FLOPs estimator: workload-aware | logical tokens/s: 75038.2 | valid tokens/s: 75038.2 | physical tokens/s: 76004.3 | loss tokens/s: 56727.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.317672E-01 | lm-global-supervised-tokens: 1.268317E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.465902E-02 | lm-token-buffer-mib: 2.363987E+02 | global_load_balancing_loss: 9.998465E-01 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11233.082 | params rms: 0.060823 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:01:05.006073 | finish at 2026-10-17 17:51:41 + [2026-10-11 16:54:37] iteration 50/ 2384 | consumed samples: 1600 | elapsed time per iteration (ms): 240576.7 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.71% | FLOPs estimator: workload-aware | logical tokens/s: 69737.5 | valid tokens/s: 69737.5 | physical tokens/s: 70013.9 | loss tokens/s: 50210.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.235597E-01 | lm-global-supervised-tokens: 1.207950E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.998382E-02 | lm-token-buffer-mib: 2.333822E+02 | global_load_balancing_loss: 9.989101E-01 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11233.016 | params rms: 0.060823 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 11:58:25.942612 | finish at 2026-10-18 04:53:03 + [2026-10-11 16:58:17] iteration 51/ 2384 | consumed samples: 1632 | elapsed time per iteration (ms): 219759.3 | throughput per GPU (TFLOP/s/GPU): 25.1 | MFU 8.03% | FLOPs estimator: workload-aware | logical tokens/s: 76343.4 | valid tokens/s: 76343.4 | physical tokens/s: 76706.2 | loss tokens/s: 55545.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.271363E-01 | lm-global-supervised-tokens: 1.220662E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.892759E-02 | lm-token-buffer-mib: 2.322447E+02 | global_load_balancing_loss: 1.001293E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11232.950 | params rms: 0.060823 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:24:58.362338 | finish at 2026-10-17 15:23:15 + [2026-10-11 17:02:02] iteration 52/ 2384 | consumed samples: 1664 | elapsed time per iteration (ms): 225181.7 | throughput per GPU (TFLOP/s/GPU): 24.4 | MFU 7.83% | FLOPs estimator: workload-aware | logical tokens/s: 74505.1 | valid tokens/s: 74505.1 | physical tokens/s: 75678.5 | loss tokens/s: 56918.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.788029E-01 | lm-global-supervised-tokens: 1.281705E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.135289E-02 | lm-token-buffer-mib: 2.331225E+02 | global_load_balancing_loss: 1.001549E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11232.885 | params rms: 0.060822 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:52:03.616517 | finish at 2026-10-17 18:54:06 + [2026-10-11 17:05:48] iteration 53/ 2384 | consumed samples: 1696 | elapsed time per iteration (ms): 225707.5 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.85% | FLOPs estimator: workload-aware | logical tokens/s: 74331.6 | valid tokens/s: 74331.6 | physical tokens/s: 75409.2 | loss tokens/s: 55944.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.571343E-01 | lm-global-supervised-tokens: 1.262710E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.611079E-02 | lm-token-buffer-mib: 2.334373E+02 | global_load_balancing_loss: 1.000478E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11232.823 | params rms: 0.060822 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 2:08:44.250813 | finish at 2026-10-17 19:14:32 + [2026-10-11 17:09:30] iteration 54/ 2384 | consumed samples: 1728 | elapsed time per iteration (ms): 221991.9 | throughput per GPU (TFLOP/s/GPU): 25.0 | MFU 8.00% | FLOPs estimator: workload-aware | logical tokens/s: 75575.7 | valid tokens/s: 75575.7 | physical tokens/s: 76174.0 | loss tokens/s: 54214.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.938254E-01 | lm-global-supervised-tokens: 1.203527E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.558147E-02 | lm-token-buffer-mib: 2.336797E+02 | global_load_balancing_loss: 1.002304E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11232.759 | params rms: 0.060822 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:40:41.078594 | finish at 2026-10-17 16:50:11 + [2026-10-11 17:13:21] iteration 55/ 2384 | consumed samples: 1760 | elapsed time per iteration (ms): 230990.4 | throughput per GPU (TFLOP/s/GPU): 25.3 | MFU 8.10% | FLOPs estimator: workload-aware | logical tokens/s: 72631.5 | valid tokens/s: 72631.5 | physical tokens/s: 73106.3 | loss tokens/s: 55184.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.645530E-01 | lm-global-supervised-tokens: 1.274705E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.443179E-02 | lm-token-buffer-mib: 2.344899E+02 | global_load_balancing_loss: 1.001004E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11232.696 | params rms: 0.060821 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 5:26:16.629561 | finish at 2026-10-17 22:39:37 + [2026-10-11 17:17:03] iteration 56/ 2384 | consumed samples: 1792 | elapsed time per iteration (ms): 222843.9 | throughput per GPU (TFLOP/s/GPU): 23.7 | MFU 7.58% | FLOPs estimator: workload-aware | logical tokens/s: 75286.7 | valid tokens/s: 75286.7 | physical tokens/s: 75644.7 | loss tokens/s: 54235.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.678661E-01 | lm-global-supervised-tokens: 1.208595E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.197685E-02 | lm-token-buffer-mib: 2.333142E+02 | global_load_balancing_loss: 1.001080E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11232.635 | params rms: 0.060821 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 0:06:20.559711 | finish at 2026-10-17 17:23:24 + [2026-10-11 17:20:53] iteration 57/ 2384 | consumed samples: 1824 | elapsed time per iteration (ms): 229558.2 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.36% | FLOPs estimator: workload-aware | logical tokens/s: 73084.8 | valid tokens/s: 73084.8 | physical tokens/s: 73816.4 | loss tokens/s: 57685.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.307198E-01 | lm-global-supervised-tokens: 1.324221E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.270923E-02 | lm-token-buffer-mib: 2.359860E+02 | global_load_balancing_loss: 9.989684E-01 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11232.574 | params rms: 0.060821 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 4:23:02.014901 | finish at 2026-10-17 21:43:55 + [2026-10-11 17:24:35] iteration 58/ 2384 | consumed samples: 1856 | elapsed time per iteration (ms): 221531.8 | throughput per GPU (TFLOP/s/GPU): 17.8 | MFU 5.70% | FLOPs estimator: workload-aware | logical tokens/s: 75732.7 | valid tokens/s: 75732.7 | physical tokens/s: 76908.3 | loss tokens/s: 59826.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.619570E-01 | lm-global-supervised-tokens: 1.325357E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.396109E-02 | lm-token-buffer-mib: 2.366273E+02 | global_load_balancing_loss: 1.000546E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11232.512 | params rms: 0.060820 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:08:03.030094 | finish at 2026-10-17 16:32:38 + [2026-10-11 17:28:16] iteration 59/ 2384 | consumed samples: 1888 | elapsed time per iteration (ms): 221489.4 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.90% | FLOPs estimator: workload-aware | logical tokens/s: 75747.1 | valid tokens/s: 75747.1 | physical tokens/s: 76724.5 | loss tokens/s: 61966.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.322217E-01 | lm-global-supervised-tokens: 1.372491E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.108568E-02 | lm-token-buffer-mib: 2.351994E+02 | global_load_balancing_loss: 1.000410E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11232.450 | params rms: 0.060820 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:02:42.966210 | finish at 2026-10-17 16:30:59 + [2026-10-11 17:32:16] iteration 60/ 2384 | consumed samples: 1920 | elapsed time per iteration (ms): 239858.1 | throughput per GPU (TFLOP/s/GPU): 17.3 | MFU 5.56% | FLOPs estimator: workload-aware | logical tokens/s: 69946.4 | valid tokens/s: 69946.4 | physical tokens/s: 70776.5 | loss tokens/s: 51515.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.044330E-01 | lm-global-supervised-tokens: 1.235637E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.502503E-02 | lm-token-buffer-mib: 2.350911E+02 | global_load_balancing_loss: 9.997380E-01 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11232.389 | params rms: 0.060820 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 10:50:30.142804 | finish at 2026-10-18 04:22:46 + [2026-10-11 17:36:00] iteration 61/ 2384 | consumed samples: 1952 | elapsed time per iteration (ms): 223901.3 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.60% | FLOPs estimator: workload-aware | logical tokens/s: 74931.1 | valid tokens/s: 74931.1 | physical tokens/s: 76024.7 | loss tokens/s: 52967.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.230019E-01 | lm-global-supervised-tokens: 1.185954E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.749394E-02 | lm-token-buffer-mib: 2.364337E+02 | global_load_balancing_loss: 1.004082E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11232.332 | params rms: 0.060819 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 0:28:42.610130 | finish at 2026-10-17 18:04:42 + [2026-10-11 17:39:41] iteration 62/ 2384 | consumed samples: 1984 | elapsed time per iteration (ms): 221425.6 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.41% | FLOPs estimator: workload-aware | logical tokens/s: 75769.0 | valid tokens/s: 75769.0 | physical tokens/s: 76721.7 | loss tokens/s: 58042.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.106827E-01 | lm-global-supervised-tokens: 1.285203E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.850157E-02 | lm-token-buffer-mib: 2.343329E+02 | global_load_balancing_loss: 1.001474E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11232.274 | params rms: 0.060819 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:49:10.325254 | finish at 2026-10-17 16:28:52 + [2026-10-11 17:43:23] iteration 63/ 2384 | consumed samples: 2016 | elapsed time per iteration (ms): 221359.2 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.45% | FLOPs estimator: workload-aware | logical tokens/s: 75791.8 | valid tokens/s: 75791.8 | physical tokens/s: 76689.5 | loss tokens/s: 58723.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.893934E-01 | lm-global-supervised-tokens: 1.299897E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.444045E-02 | lm-token-buffer-mib: 2.360183E+02 | global_load_balancing_loss: 9.993465E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11232.216 | params rms: 0.060819 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:42:54.697115 | finish at 2026-10-17 16:26:17 + [2026-10-11 17:47:06] iteration 64/ 2384 | consumed samples: 2048 | elapsed time per iteration (ms): 223290.5 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.98% | FLOPs estimator: workload-aware | logical tokens/s: 75136.2 | valid tokens/s: 75136.2 | physical tokens/s: 75794.6 | loss tokens/s: 59293.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.610875E-01 | lm-global-supervised-tokens: 1.323959E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.672171E-02 | lm-token-buffer-mib: 2.353088E+02 | global_load_balancing_loss: 9.996882E-01 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11232.159 | params rms: 0.060818 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:53:53.955956 | finish at 2026-10-17 17:41:00 + [2026-10-11 17:50:49] iteration 65/ 2384 | consumed samples: 2080 | elapsed time per iteration (ms): 223132.5 | throughput per GPU (TFLOP/s/GPU): 27.2 | MFU 8.71% | FLOPs estimator: workload-aware | logical tokens/s: 75189.4 | valid tokens/s: 75189.4 | physical tokens/s: 75427.2 | loss tokens/s: 56771.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.839136E-01 | lm-global-supervised-tokens: 1.266760E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.523274E-02 | lm-token-buffer-mib: 2.328889E+02 | global_load_balancing_loss: 9.990514E-01 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11232.099 | params rms: 0.060818 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:44:04.330375 | finish at 2026-10-17 17:34:53 + [2026-10-11 17:54:33] iteration 66/ 2384 | consumed samples: 2112 | elapsed time per iteration (ms): 223993.5 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.13% | FLOPs estimator: workload-aware | logical tokens/s: 74900.3 | valid tokens/s: 74900.3 | physical tokens/s: 75445.6 | loss tokens/s: 56056.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.422974E-01 | lm-global-supervised-tokens: 1.255621E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.349815E-02 | lm-token-buffer-mib: 2.345027E+02 | global_load_balancing_loss: 1.000912E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11232.033 | params rms: 0.060818 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 0:13:36.831851 | finish at 2026-10-17 18:08:10 + [2026-10-11 17:58:17] iteration 67/ 2384 | consumed samples: 2144 | elapsed time per iteration (ms): 224326.6 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.05% | FLOPs estimator: workload-aware | logical tokens/s: 74789.1 | valid tokens/s: 74789.0 | physical tokens/s: 75382.1 | loss tokens/s: 56142.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.238558E-01 | lm-global-supervised-tokens: 1.259421E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.437055E-02 | lm-token-buffer-mib: 2.349483E+02 | global_load_balancing_loss: 1.002108E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11231.966 | params rms: 0.060817 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 0:22:44.663321 | finish at 2026-10-17 18:21:02 + [2026-10-11 18:02:04] iteration 68/ 2384 | consumed samples: 2176 | elapsed time per iteration (ms): 226374.2 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.78% | FLOPs estimator: workload-aware | logical tokens/s: 74112.7 | valid tokens/s: 74112.7 | physical tokens/s: 74877.8 | loss tokens/s: 56575.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.539176E-01 | lm-global-supervised-tokens: 1.280730E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.055442E-02 | lm-token-buffer-mib: 2.333818E+02 | global_load_balancing_loss: 1.000516E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11231.902 | params rms: 0.060817 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:38:02.538666 | finish at 2026-10-17 19:40:06 + [2026-10-11 18:05:47] iteration 69/ 2384 | consumed samples: 2208 | elapsed time per iteration (ms): 222931.7 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.65% | FLOPs estimator: workload-aware | logical tokens/s: 75257.1 | valid tokens/s: 75257.1 | physical tokens/s: 76321.1 | loss tokens/s: 52948.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.417812E-01 | lm-global-supervised-tokens: 1.180386E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.360563E-02 | lm-token-buffer-mib: 2.347101E+02 | global_load_balancing_loss: 1.000982E+00 | loss scale: 1.0 | grad norm: 0.114 | params norm: 11231.841 | params rms: 0.060817 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:21:26.963303 | finish at 2026-10-17 17:27:14 + [2026-10-11 18:09:44] iteration 70/ 2384 | consumed samples: 2240 | elapsed time per iteration (ms): 237136.2 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.57% | FLOPs estimator: workload-aware | logical tokens/s: 70749.2 | valid tokens/s: 70749.2 | physical tokens/s: 71425.6 | loss tokens/s: 51513.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.772311E-01 | lm-global-supervised-tokens: 1.221563E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.487821E-02 | lm-token-buffer-mib: 2.327875E+02 | global_load_balancing_loss: 1.001385E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11231.780 | params rms: 0.060816 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 8:25:33.107655 | finish at 2026-10-18 02:35:17 + [2026-10-11 18:13:27] iteration 71/ 2384 | consumed samples: 2272 | elapsed time per iteration (ms): 222785.4 | throughput per GPU (TFLOP/s/GPU): 18.9 | MFU 6.06% | FLOPs estimator: workload-aware | logical tokens/s: 75306.6 | valid tokens/s: 75306.6 | physical tokens/s: 76353.7 | loss tokens/s: 59690.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.186094E-01 | lm-global-supervised-tokens: 1.329816E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.369280E-02 | lm-token-buffer-mib: 2.354453E+02 | global_load_balancing_loss: 1.000712E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11231.720 | params rms: 0.060816 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:08:22.529634 | finish at 2026-10-17 17:21:49 + [2026-10-11 18:17:08] iteration 72/ 2384 | consumed samples: 2304 | elapsed time per iteration (ms): 221139.2 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.04% | FLOPs estimator: workload-aware | logical tokens/s: 75867.2 | valid tokens/s: 75867.2 | physical tokens/s: 76766.4 | loss tokens/s: 56472.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.105680E-01 | lm-global-supervised-tokens: 1.248833E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.455913E-02 | lm-token-buffer-mib: 2.342398E+02 | global_load_balancing_loss: 1.000005E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11231.658 | params rms: 0.060816 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:01:13.751511 | finish at 2026-10-17 16:18:21 + [2026-10-11 18:20:47] iteration 73/ 2384 | consumed samples: 2336 | elapsed time per iteration (ms): 219764.8 | throughput per GPU (TFLOP/s/GPU): 19.5 | MFU 6.26% | FLOPs estimator: workload-aware | logical tokens/s: 76341.6 | valid tokens/s: 76341.6 | physical tokens/s: 76824.8 | loss tokens/s: 59970.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.629897E-01 | lm-global-supervised-tokens: 1.317930E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.378840E-02 | lm-token-buffer-mib: 2.336733E+02 | global_load_balancing_loss: 1.000804E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11231.595 | params rms: 0.060815 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:04:36.417152 | finish at 2026-10-17 15:25:24 + [2026-10-11 18:24:29] iteration 74/ 2384 | consumed samples: 2368 | elapsed time per iteration (ms): 221180.8 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.08% | FLOPs estimator: workload-aware | logical tokens/s: 75852.9 | valid tokens/s: 75852.9 | physical tokens/s: 76391.1 | loss tokens/s: 56387.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.728871E-01 | lm-global-supervised-tokens: 1.247187E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.935239E-02 | lm-token-buffer-mib: 2.333587E+02 | global_load_balancing_loss: 1.001362E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11231.535 | params rms: 0.060815 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:55:27.688115 | finish at 2026-10-17 16:19:56 + [2026-10-11 18:28:09] iteration 75/ 2384 | consumed samples: 2400 | elapsed time per iteration (ms): 220650.3 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.36% | FLOPs estimator: workload-aware | logical tokens/s: 76035.2 | valid tokens/s: 76035.2 | physical tokens/s: 76681.0 | loss tokens/s: 58672.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.899670E-01 | lm-global-supervised-tokens: 1.294602E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.005864E-02 | lm-token-buffer-mib: 2.333930E+02 | global_load_balancing_loss: 1.000476E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11231.475 | params rms: 0.060815 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:31:21.594508 | finish at 2026-10-17 15:59:31 + [2026-10-11 18:31:49] iteration 76/ 2384 | consumed samples: 2432 | elapsed time per iteration (ms): 219540.4 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 7.00% | FLOPs estimator: workload-aware | logical tokens/s: 76419.6 | valid tokens/s: 76419.6 | physical tokens/s: 77199.5 | loss tokens/s: 56004.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.277272E-01 | lm-global-supervised-tokens: 1.229525E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.664651E-02 | lm-token-buffer-mib: 2.339110E+02 | global_load_balancing_loss: 1.001127E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11231.414 | params rms: 0.060814 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:44:59.276281 | finish at 2026-10-17 15:16:48 + [2026-10-11 18:35:30] iteration 77/ 2384 | consumed samples: 2464 | elapsed time per iteration (ms): 221302.9 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.90% | FLOPs estimator: workload-aware | logical tokens/s: 75811.0 | valid tokens/s: 75811.0 | physical tokens/s: 76885.7 | loss tokens/s: 58454.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.361924E-01 | lm-global-supervised-tokens: 1.293623E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.015193E-02 | lm-token-buffer-mib: 2.352722E+02 | global_load_balancing_loss: 1.001489E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11231.353 | params rms: 0.060814 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:49:05.872980 | finish at 2026-10-17 16:24:36 + [2026-10-11 18:39:17] iteration 78/ 2384 | consumed samples: 2496 | elapsed time per iteration (ms): 227194.6 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.23% | FLOPs estimator: workload-aware | logical tokens/s: 73845.0 | valid tokens/s: 73845.0 | physical tokens/s: 74590.2 | loss tokens/s: 54919.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.122805E-01 | lm-global-supervised-tokens: 1.247736E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.193914E-02 | lm-token-buffer-mib: 2.346710E+02 | global_load_balancing_loss: 9.997762E-01 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11231.293 | params rms: 0.060814 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:31:50.678569 | finish at 2026-10-17 20:11:08 + [2026-10-11 18:43:00] iteration 79/ 2384 | consumed samples: 2528 | elapsed time per iteration (ms): 222234.8 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.35% | FLOPs estimator: workload-aware | logical tokens/s: 75493.1 | valid tokens/s: 75493.1 | physical tokens/s: 75976.4 | loss tokens/s: 59353.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.396019E-01 | lm-global-supervised-tokens: 1.319048E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.269524E-02 | lm-token-buffer-mib: 2.325827E+02 | global_load_balancing_loss: 9.994640E-01 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11231.232 | params rms: 0.060813 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:17:31.177961 | finish at 2026-10-17 17:00:31 + [2026-10-11 18:46:58] iteration 80/ 2384 | consumed samples: 2560 | elapsed time per iteration (ms): 238242.4 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.84% | FLOPs estimator: workload-aware | logical tokens/s: 70420.7 | valid tokens/s: 70420.7 | physical tokens/s: 71262.9 | loss tokens/s: 56563.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.715167E-01 | lm-global-supervised-tokens: 1.347582E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.161023E-02 | lm-token-buffer-mib: 2.341868E+02 | global_load_balancing_loss: 9.989673E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11231.170 | params rms: 0.060813 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 8:28:30.488892 | finish at 2026-10-18 03:15:28 +times across ranks (ms): + save-checkpoint: + rank 0: 105960.05 + rank 1: 105960.07 + rank 2: 105960.06 + rank 3: 105959.99 + rank 4: 105959.97 + rank 5: 105960.07 + rank 6: 105960.01 + rank 7: 105959.98 + rank 8: 105962.69 + rank 9: 105962.61 + rank 10: 105962.61 + rank 11: 105962.68 + rank 12: 105962.70 + rank 13: 105962.69 + rank 14: 105962.70 + rank 15: 105962.70 + rank 16: 105960.41 + rank 17: 105960.31 + rank 18: 105960.31 + rank 19: 105960.32 + rank 20: 105960.33 + rank 21: 105960.38 + rank 22: 105960.34 + rank 23: 105960.32 + rank 24: 105959.09 + rank 25: 105959.09 + rank 26: 105959.08 + rank 27: 105959.09 + rank 28: 105959.06 + rank 29: 105959.12 + rank 30: 105959.09 + rank 31: 105959.08 + rank 32: 105957.57 + rank 33: 105957.56 + rank 34: 105957.57 + rank 35: 105957.57 + rank 36: 105957.57 + rank 37: 105957.57 + rank 38: 105957.68 + rank 39: 105957.60 + rank 40: 105958.96 + rank 41: 105959.12 + rank 42: 105958.98 + rank 43: 105958.97 + rank 44: 105959.02 + rank 45: 105959.04 + rank 46: 105959.02 + rank 47: 105958.95 + rank 48: 105959.84 + rank 49: 105959.85 + rank 50: 105959.87 + rank 51: 105959.84 + rank 52: 105959.84 + rank 53: 105959.83 + rank 54: 105959.84 + rank 55: 105959.84 + rank 56: 105958.18 + rank 57: 105958.20 + rank 58: 105958.20 + rank 59: 105958.20 + rank 60: 105958.20 + rank 61: 105958.18 + rank 62: 105958.17 + rank 63: 105958.17 + [2026-10-11 18:52:21] iteration 81/ 2384 | consumed samples: 2592 | elapsed time per iteration (ms): 217125.4 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.24% | FLOPs estimator: workload-aware | logical tokens/s: 77269.5 | valid tokens/s: 77269.5 | physical tokens/s: 78076.8 | loss tokens/s: 61044.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.019277E-01 | lm-global-supervised-tokens: 1.325423E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.973375E-02 | lm-token-buffer-mib: 2.339876E+02 | global_load_balancing_loss: 1.003084E+00 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11231.105 | params rms: 0.060813 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:53:59.682145 | finish at 2026-10-17 13:46:21 + [2026-10-11 18:56:10] iteration 82/ 2384 | consumed samples: 2624 | elapsed time per iteration (ms): 229065.9 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.65% | FLOPs estimator: workload-aware | logical tokens/s: 73241.7 | valid tokens/s: 73241.7 | physical tokens/s: 74169.2 | loss tokens/s: 56003.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.818955E-01 | lm-global-supervised-tokens: 1.282844E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.483049E-02 | lm-token-buffer-mib: 2.353244E+02 | global_load_balancing_loss: 9.997398E-01 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11231.039 | params rms: 0.060812 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 2:28:29.795853 | finish at 2026-10-17 21:24:40 + [2026-10-11 18:59:55] iteration 83/ 2384 | consumed samples: 2656 | elapsed time per iteration (ms): 224734.9 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.53% | FLOPs estimator: workload-aware | logical tokens/s: 74653.3 | valid tokens/s: 74653.3 | physical tokens/s: 75717.3 | loss tokens/s: 58796.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.765146E-01 | lm-global-supervised-tokens: 1.321370E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.351702E-02 | lm-token-buffer-mib: 2.361575E+02 | global_load_balancing_loss: 9.997645E-01 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11230.972 | params rms: 0.060812 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:38:34.924799 | finish at 2026-10-17 18:38:30 + [2026-10-11 19:03:40] iteration 84/ 2384 | consumed samples: 2688 | elapsed time per iteration (ms): 224964.9 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.89% | FLOPs estimator: workload-aware | logical tokens/s: 74576.9 | valid tokens/s: 74576.9 | physical tokens/s: 75279.7 | loss tokens/s: 56671.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.815928E-01 | lm-global-supervised-tokens: 1.274904E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.879406E-02 | lm-token-buffer-mib: 2.334554E+02 | global_load_balancing_loss: 1.000990E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11230.906 | params rms: 0.060812 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:43:39.382453 | finish at 2026-10-17 18:47:19 + [2026-10-11 19:07:21] iteration 85/ 2384 | consumed samples: 2720 | elapsed time per iteration (ms): 221305.4 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.90% | FLOPs estimator: workload-aware | logical tokens/s: 75810.0 | valid tokens/s: 75810.0 | physical tokens/s: 76838.3 | loss tokens/s: 55896.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.539368E-01 | lm-global-supervised-tokens: 1.237013E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.373177E-02 | lm-token-buffer-mib: 2.343750E+02 | global_load_balancing_loss: 1.001164E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11230.838 | params rms: 0.060811 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:19:41.213568 | finish at 2026-10-17 16:27:02 + [2026-10-11 19:11:04] iteration 86/ 2384 | consumed samples: 2752 | elapsed time per iteration (ms): 223043.8 | throughput per GPU (TFLOP/s/GPU): 19.6 | MFU 6.27% | FLOPs estimator: workload-aware | logical tokens/s: 75219.3 | valid tokens/s: 75219.3 | physical tokens/s: 76730.7 | loss tokens/s: 58014.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.451549E-01 | lm-global-supervised-tokens: 1.293969E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.238102E-02 | lm-token-buffer-mib: 2.388286E+02 | global_load_balancing_loss: 1.000505E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11230.768 | params rms: 0.060811 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:22:34.755668 | finish at 2026-10-17 17:33:39 + [2026-10-11 19:14:43] iteration 87/ 2384 | consumed samples: 2784 | elapsed time per iteration (ms): 218927.6 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.03% | FLOPs estimator: workload-aware | logical tokens/s: 76633.4 | valid tokens/s: 76633.4 | physical tokens/s: 77087.6 | loss tokens/s: 60976.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.852182E-01 | lm-global-supervised-tokens: 1.334953E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.316172E-02 | lm-token-buffer-mib: 2.335181E+02 | global_load_balancing_loss: 1.000250E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11230.702 | params rms: 0.060811 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:41:16.663032 | finish at 2026-10-17 14:56:00 + [2026-10-11 19:18:24] iteration 88/ 2384 | consumed samples: 2816 | elapsed time per iteration (ms): 221290.1 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.89% | FLOPs estimator: workload-aware | logical tokens/s: 75815.5 | valid tokens/s: 75815.5 | physical tokens/s: 76730.3 | loss tokens/s: 55410.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.881902E-01 | lm-global-supervised-tokens: 1.226176E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.365034E-02 | lm-token-buffer-mib: 2.349746E+02 | global_load_balancing_loss: 1.001308E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11230.636 | params rms: 0.060810 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:08:02.047380 | finish at 2026-10-17 16:26:26 + [2026-10-11 19:22:11] iteration 89/ 2384 | consumed samples: 2848 | elapsed time per iteration (ms): 226612.5 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.32% | FLOPs estimator: workload-aware | logical tokens/s: 74034.8 | valid tokens/s: 74034.8 | physical tokens/s: 74771.6 | loss tokens/s: 58947.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.437210E-01 | lm-global-supervised-tokens: 1.335834E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.465213E-02 | lm-token-buffer-mib: 2.346078E+02 | global_load_balancing_loss: 1.000398E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11230.571 | params rms: 0.060810 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 0:27:55.782598 | finish at 2026-10-17 19:50:07 + [2026-10-11 19:26:14] iteration 90/ 2384 | consumed samples: 2880 | elapsed time per iteration (ms): 243236.2 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.93% | FLOPs estimator: workload-aware | logical tokens/s: 68974.9 | valid tokens/s: 68974.9 | physical tokens/s: 69602.5 | loss tokens/s: 50679.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.733403E-01 | lm-global-supervised-tokens: 1.232710E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.262643E-02 | lm-token-buffer-mib: 2.342825E+02 | global_load_balancing_loss: 1.001585E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11230.505 | params rms: 0.060810 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 10:59:43.880754 | finish at 2026-10-18 06:25:58 + [2026-10-11 19:30:02] iteration 91/ 2384 | consumed samples: 2912 | elapsed time per iteration (ms): 228375.8 | throughput per GPU (TFLOP/s/GPU): 17.6 | MFU 5.66% | FLOPs estimator: workload-aware | logical tokens/s: 73463.2 | valid tokens/s: 73463.2 | physical tokens/s: 74523.3 | loss tokens/s: 59455.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.236550E-01 | lm-global-supervised-tokens: 1.357830E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.747432E-02 | lm-token-buffer-mib: 2.359829E+02 | global_load_balancing_loss: 1.000728E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11230.438 | params rms: 0.060809 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:27:45.671436 | finish at 2026-10-17 20:57:48 + [2026-10-11 19:33:43] iteration 92/ 2384 | consumed samples: 2944 | elapsed time per iteration (ms): 220997.0 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.57% | FLOPs estimator: workload-aware | logical tokens/s: 75916.0 | valid tokens/s: 75916.0 | physical tokens/s: 77069.5 | loss tokens/s: 58515.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.476588E-01 | lm-global-supervised-tokens: 1.293172E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.096402E-02 | lm-token-buffer-mib: 2.344597E+02 | global_load_balancing_loss: 1.000704E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11230.373 | params rms: 0.060809 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:42:05.187341 | finish at 2026-10-17 16:15:49 + [2026-10-11 19:37:29] iteration 93/ 2384 | consumed samples: 2976 | elapsed time per iteration (ms): 225670.0 | throughput per GPU (TFLOP/s/GPU): 24.2 | MFU 7.76% | FLOPs estimator: workload-aware | logical tokens/s: 74343.7 | valid tokens/s: 74343.7 | physical tokens/s: 74921.4 | loss tokens/s: 54920.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.514231E-01 | lm-global-supervised-tokens: 1.239392E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.932313E-02 | lm-token-buffer-mib: 2.337751E+02 | global_load_balancing_loss: 1.002926E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11230.307 | params rms: 0.060808 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:36:50.021519 | finish at 2026-10-17 19:14:19 + [2026-10-11 19:41:09] iteration 94/ 2384 | consumed samples: 3008 | elapsed time per iteration (ms): 219764.9 | throughput per GPU (TFLOP/s/GPU): 23.9 | MFU 7.66% | FLOPs estimator: workload-aware | logical tokens/s: 76341.6 | valid tokens/s: 76341.6 | physical tokens/s: 76669.6 | loss tokens/s: 58850.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.429054E-01 | lm-global-supervised-tokens: 1.293327E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.935369E-02 | lm-token-buffer-mib: 2.332067E+02 | global_load_balancing_loss: 1.002276E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11230.240 | params rms: 0.060808 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:47:41.632395 | finish at 2026-10-17 15:28:51 + [2026-10-11 19:44:51] iteration 95/ 2384 | consumed samples: 3040 | elapsed time per iteration (ms): 222173.1 | throughput per GPU (TFLOP/s/GPU): 19.3 | MFU 6.17% | FLOPs estimator: workload-aware | logical tokens/s: 75514.1 | valid tokens/s: 75514.1 | physical tokens/s: 76364.2 | loss tokens/s: 57743.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.383941E-01 | lm-global-supervised-tokens: 1.282900E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.605413E-02 | lm-token-buffer-mib: 2.363120E+02 | global_load_balancing_loss: 9.994134E-01 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11230.174 | params rms: 0.060808 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:15:54.252629 | finish at 2026-10-17 17:00:45 + [2026-10-11 19:48:32] iteration 96/ 2384 | consumed samples: 3072 | elapsed time per iteration (ms): 221027.0 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.40% | FLOPs estimator: workload-aware | logical tokens/s: 75905.6 | valid tokens/s: 75905.6 | physical tokens/s: 76265.1 | loss tokens/s: 59787.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.095393E-01 | lm-global-supervised-tokens: 1.321454E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.206619E-02 | lm-token-buffer-mib: 2.329427E+02 | global_load_balancing_loss: 1.000370E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11230.108 | params rms: 0.060807 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:28:29.850075 | finish at 2026-10-17 16:17:02 + [2026-10-11 19:52:21] iteration 97/ 2384 | consumed samples: 3104 | elapsed time per iteration (ms): 228520.8 | throughput per GPU (TFLOP/s/GPU): 26.8 | MFU 8.58% | FLOPs estimator: workload-aware | logical tokens/s: 73416.4 | valid tokens/s: 73416.4 | physical tokens/s: 74337.1 | loss tokens/s: 53782.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.766208E-01 | lm-global-supervised-tokens: 1.229049E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.490897E-02 | lm-token-buffer-mib: 2.359875E+02 | global_load_balancing_loss: 1.000557E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11230.041 | params rms: 0.060807 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:10:27.180548 | finish at 2026-10-17 21:02:48 + [2026-10-11 19:56:04] iteration 98/ 2384 | consumed samples: 3136 | elapsed time per iteration (ms): 223800.8 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.51% | FLOPs estimator: workload-aware | logical tokens/s: 74964.9 | valid tokens/s: 74964.9 | physical tokens/s: 75358.2 | loss tokens/s: 56997.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.912655E-01 | lm-global-supervised-tokens: 1.275606E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.788471E-02 | lm-token-buffer-mib: 2.329099E+02 | global_load_balancing_loss: 1.000011E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11229.975 | params rms: 0.060807 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:06:48.589208 | finish at 2026-10-17 18:02:53 + [2026-10-11 19:59:46] iteration 99/ 2384 | consumed samples: 3168 | elapsed time per iteration (ms): 221660.6 | throughput per GPU (TFLOP/s/GPU): 19.3 | MFU 6.18% | FLOPs estimator: workload-aware | logical tokens/s: 75688.6 | valid tokens/s: 75688.6 | physical tokens/s: 76841.7 | loss tokens/s: 61206.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.695986E-01 | lm-global-supervised-tokens: 1.356704E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.687880E-02 | lm-token-buffer-mib: 2.353491E+02 | global_load_balancing_loss: 1.000999E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11229.912 | params rms: 0.060806 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:41:34.519365 | finish at 2026-10-17 16:41:21 + [2026-10-11 20:03:44] iteration 100/ 2384 | consumed samples: 3200 | elapsed time per iteration (ms): 237919.7 | throughput per GPU (TFLOP/s/GPU): 19.9 | MFU 6.37% | FLOPs estimator: workload-aware | logical tokens/s: 70516.2 | valid tokens/s: 70516.2 | physical tokens/s: 71064.3 | loss tokens/s: 53180.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.981089E-01 | lm-global-supervised-tokens: 1.265271E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.611489E-02 | lm-token-buffer-mib: 2.346313E+02 | global_load_balancing_loss: 1.000186E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11229.848 | params rms: 0.060806 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 6:56:48.528153 | finish at 2026-10-18 03:00:33 + [2026-10-11 20:07:25] iteration 101/ 2384 | consumed samples: 3232 | elapsed time per iteration (ms): 221250.2 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.59% | FLOPs estimator: workload-aware | logical tokens/s: 75829.1 | valid tokens/s: 75829.1 | physical tokens/s: 76503.1 | loss tokens/s: 56938.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.319414E-01 | lm-global-supervised-tokens: 1.259754E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.485626E-02 | lm-token-buffer-mib: 2.347661E+02 | global_load_balancing_loss: 9.999180E-01 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11229.786 | params rms: 0.060806 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:18:34.220828 | finish at 2026-10-17 16:25:59 + [2026-10-11 20:11:06] iteration 102/ 2384 | consumed samples: 3264 | elapsed time per iteration (ms): 220799.9 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.02% | FLOPs estimator: workload-aware | logical tokens/s: 75983.6 | valid tokens/s: 75983.6 | physical tokens/s: 76320.9 | loss tokens/s: 54802.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.650785E-01 | lm-global-supervised-tokens: 1.210027E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.913318E-02 | lm-token-buffer-mib: 2.333345E+02 | global_load_balancing_loss: 1.000279E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11229.721 | params rms: 0.060805 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:57:45.339281 | finish at 2026-10-17 16:08:51 + [2026-10-11 20:14:56] iteration 103/ 2384 | consumed samples: 3296 | elapsed time per iteration (ms): 229619.8 | throughput per GPU (TFLOP/s/GPU): 18.8 | MFU 6.03% | FLOPs estimator: workload-aware | logical tokens/s: 73065.2 | valid tokens/s: 73065.2 | physical tokens/s: 74324.7 | loss tokens/s: 59450.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.284775E-01 | lm-global-supervised-tokens: 1.365106E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.128989E-02 | lm-token-buffer-mib: 2.349807E+02 | global_load_balancing_loss: 1.001834E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11229.654 | params rms: 0.060805 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:29:22.729202 | finish at 2026-10-17 21:44:18 + [2026-10-11 20:18:37] iteration 104/ 2384 | consumed samples: 3328 | elapsed time per iteration (ms): 221789.8 | throughput per GPU (TFLOP/s/GPU): 23.5 | MFU 7.54% | FLOPs estimator: workload-aware | logical tokens/s: 75644.6 | valid tokens/s: 75644.6 | physical tokens/s: 76363.8 | loss tokens/s: 59328.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.687084E-01 | lm-global-supervised-tokens: 1.315848E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.416581E-02 | lm-token-buffer-mib: 2.342785E+02 | global_load_balancing_loss: 1.002093E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11229.591 | params rms: 0.060805 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:28:00.742750 | finish at 2026-10-17 16:46:38 + [2026-10-11 20:22:30] iteration 105/ 2384 | consumed samples: 3360 | elapsed time per iteration (ms): 232548.8 | throughput per GPU (TFLOP/s/GPU): 25.7 | MFU 8.25% | FLOPs estimator: workload-aware | logical tokens/s: 72144.7 | valid tokens/s: 72144.7 | physical tokens/s: 72809.8 | loss tokens/s: 55150.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 1.295711E+00 | lm-global-supervised-tokens: 1.282510E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.627463E-02 | lm-token-buffer-mib: 2.346182E+02 | global_load_balancing_loss: 1.002616E+00 | loss scale: 1.0 | grad norm: 19.985 | params norm: 11229.525 | params rms: 0.060804 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 3:12:58.766800 | finish at 2026-10-17 23:35:29 + [2026-10-11 20:26:13] iteration 106/ 2384 | consumed samples: 3392 | elapsed time per iteration (ms): 223374.8 | throughput per GPU (TFLOP/s/GPU): 24.0 | MFU 7.69% | FLOPs estimator: workload-aware | logical tokens/s: 75107.8 | valid tokens/s: 75107.8 | physical tokens/s: 75582.1 | loss tokens/s: 53197.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.872565E-01 | lm-global-supervised-tokens: 1.188306E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.749618E-02 | lm-token-buffer-mib: 2.332198E+02 | global_load_balancing_loss: 1.003287E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11229.441 | params rms: 0.060804 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:20:47.813877 | finish at 2026-10-17 17:47:01 + [2026-10-11 20:29:54] iteration 107/ 2384 | consumed samples: 3424 | elapsed time per iteration (ms): 220966.5 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.68% | FLOPs estimator: workload-aware | logical tokens/s: 75926.4 | valid tokens/s: 75926.4 | physical tokens/s: 76515.3 | loss tokens/s: 58284.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.818594E-01 | lm-global-supervised-tokens: 1.287887E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.921105E-02 | lm-token-buffer-mib: 2.330961E+02 | global_load_balancing_loss: 1.003426E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11229.379 | params rms: 0.060803 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:45:40.833519 | finish at 2026-10-17 16:15:35 + [2026-10-11 20:33:37] iteration 108/ 2384 | consumed samples: 3456 | elapsed time per iteration (ms): 222420.4 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.41% | FLOPs estimator: workload-aware | logical tokens/s: 75430.0 | valid tokens/s: 75430.0 | physical tokens/s: 76177.5 | loss tokens/s: 57148.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.803079E-01 | lm-global-supervised-tokens: 1.271095E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.353657E-02 | lm-token-buffer-mib: 2.339656E+02 | global_load_balancing_loss: 1.001889E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11229.331 | params rms: 0.060803 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:37:08.780802 | finish at 2026-10-17 17:10:46 + [2026-10-11 20:37:21] iteration 109/ 2384 | consumed samples: 3488 | elapsed time per iteration (ms): 224053.2 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.21% | FLOPs estimator: workload-aware | logical tokens/s: 74880.4 | valid tokens/s: 74880.4 | physical tokens/s: 76407.3 | loss tokens/s: 54090.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.201604E-01 | lm-global-supervised-tokens: 1.211910E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.716265E-02 | lm-token-buffer-mib: 2.365849E+02 | global_load_balancing_loss: 1.002721E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11229.291 | params rms: 0.060803 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:35:21.139580 | finish at 2026-10-17 18:12:42 + [2026-10-11 20:41:22] iteration 110/ 2384 | consumed samples: 3520 | elapsed time per iteration (ms): 241535.0 | throughput per GPU (TFLOP/s/GPU): 19.0 | MFU 6.08% | FLOPs estimator: workload-aware | logical tokens/s: 69460.6 | valid tokens/s: 69460.6 | physical tokens/s: 70318.8 | loss tokens/s: 53753.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.048456E-01 | lm-global-supervised-tokens: 1.298324E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.792133E-02 | lm-token-buffer-mib: 2.363940E+02 | global_load_balancing_loss: 9.992131E-01 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11229.258 | params rms: 0.060803 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 8:34:10.568509 | finish at 2026-10-18 05:15:33 + [2026-10-11 20:45:10] iteration 111/ 2384 | consumed samples: 3552 | elapsed time per iteration (ms): 228025.2 | throughput per GPU (TFLOP/s/GPU): 24.3 | MFU 7.80% | FLOPs estimator: workload-aware | logical tokens/s: 73576.0 | valid tokens/s: 73576.0 | physical tokens/s: 74056.1 | loss tokens/s: 59972.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.915825E-01 | lm-global-supervised-tokens: 1.367516E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.388277E-02 | lm-token-buffer-mib: 2.335115E+02 | global_load_balancing_loss: 1.001333E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11229.229 | params rms: 0.060803 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:58:21.224074 | finish at 2026-10-17 20:43:32 + [2026-10-11 20:48:51] iteration 112/ 2384 | consumed samples: 3584 | elapsed time per iteration (ms): 220586.7 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.23% | FLOPs estimator: workload-aware | logical tokens/s: 76057.0 | valid tokens/s: 76057.0 | physical tokens/s: 76761.7 | loss tokens/s: 60616.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.360054E-01 | lm-global-supervised-tokens: 1.337126E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.232684E-02 | lm-token-buffer-mib: 2.332435E+02 | global_load_balancing_loss: 9.984341E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11229.205 | params rms: 0.060802 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:12:53.081985 | finish at 2026-10-17 16:01:44 + [2026-10-11 20:52:32] iteration 113/ 2384 | consumed samples: 3616 | elapsed time per iteration (ms): 220828.6 | throughput per GPU (TFLOP/s/GPU): 24.5 | MFU 7.85% | FLOPs estimator: workload-aware | logical tokens/s: 75973.9 | valid tokens/s: 75973.8 | physical tokens/s: 76328.1 | loss tokens/s: 55826.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.979640E-01 | lm-global-supervised-tokens: 1.232804E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.050495E-02 | lm-token-buffer-mib: 2.335573E+02 | global_load_balancing_loss: 1.001771E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11229.181 | params rms: 0.060802 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:18:21.642692 | finish at 2026-10-17 16:10:53 + [2026-10-11 20:56:17] iteration 114/ 2384 | consumed samples: 3648 | elapsed time per iteration (ms): 225513.7 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.15% | FLOPs estimator: workload-aware | logical tokens/s: 74395.5 | valid tokens/s: 74395.5 | physical tokens/s: 75068.2 | loss tokens/s: 52875.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.417013E-01 | lm-global-supervised-tokens: 1.192417E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.153073E-02 | lm-token-buffer-mib: 2.344773E+02 | global_load_balancing_loss: 9.993522E-01 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11229.157 | params rms: 0.060802 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:11:56.042733 | finish at 2026-10-17 19:08:13 + [2026-10-11 21:00:01] iteration 115/ 2384 | consumed samples: 3680 | elapsed time per iteration (ms): 223836.1 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.31% | FLOPs estimator: workload-aware | logical tokens/s: 74953.1 | valid tokens/s: 74953.1 | physical tokens/s: 75492.7 | loss tokens/s: 60480.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.639861E-01 | lm-global-supervised-tokens: 1.353760E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.598996E-02 | lm-token-buffer-mib: 2.340760E+02 | global_load_balancing_loss: 1.000683E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11229.131 | params rms: 0.060802 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:04:44.043509 | finish at 2026-10-17 18:04:45 + [2026-10-11 21:03:44] iteration 116/ 2384 | consumed samples: 3712 | elapsed time per iteration (ms): 222806.5 | throughput per GPU (TFLOP/s/GPU): 24.6 | MFU 7.88% | FLOPs estimator: workload-aware | logical tokens/s: 75299.3 | valid tokens/s: 75299.3 | physical tokens/s: 75774.6 | loss tokens/s: 58488.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.893243E-01 | lm-global-supervised-tokens: 1.303157E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.841263E-02 | lm-token-buffer-mib: 2.341477E+02 | global_load_balancing_loss: 1.000734E+00 | loss scale: 1.0 | grad norm: 0.338 | params norm: 11229.102 | params rms: 0.060802 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:22:05.056469 | finish at 2026-10-17 17:25:49 + [2026-10-11 21:07:26] iteration 117/ 2384 | consumed samples: 3744 | elapsed time per iteration (ms): 221905.5 | throughput per GPU (TFLOP/s/GPU): 19.7 | MFU 6.30% | FLOPs estimator: workload-aware | logical tokens/s: 75605.2 | valid tokens/s: 75605.2 | physical tokens/s: 76159.3 | loss tokens/s: 56764.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.106249E-01 | lm-global-supervised-tokens: 1.259637E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.466236E-02 | lm-token-buffer-mib: 2.343653E+02 | global_load_balancing_loss: 9.963008E-01 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11229.058 | params rms: 0.060802 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:44:19.748895 | finish at 2026-10-17 16:51:46 + [2026-10-11 21:11:08] iteration 118/ 2384 | consumed samples: 3776 | elapsed time per iteration (ms): 222294.5 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.26% | FLOPs estimator: workload-aware | logical tokens/s: 75472.8 | valid tokens/s: 75472.8 | physical tokens/s: 75942.8 | loss tokens/s: 60214.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.254784E-01 | lm-global-supervised-tokens: 1.338530E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.141220E-02 | lm-token-buffer-mib: 2.333490E+02 | global_load_balancing_loss: 1.001811E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11229.022 | params rms: 0.060802 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:55:19.248113 | finish at 2026-10-17 17:06:27 + [2026-10-11 21:14:55] iteration 119/ 2384 | consumed samples: 3808 | elapsed time per iteration (ms): 226752.3 | throughput per GPU (TFLOP/s/GPU): 17.2 | MFU 5.52% | FLOPs estimator: workload-aware | logical tokens/s: 73989.1 | valid tokens/s: 73989.1 | physical tokens/s: 75630.5 | loss tokens/s: 56264.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.724425E-01 | lm-global-supervised-tokens: 1.275819E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.805581E-02 | lm-token-buffer-mib: 2.348567E+02 | global_load_balancing_loss: 1.000255E+00 | loss scale: 1.0 | grad norm: 0.280 | params norm: 11228.992 | params rms: 0.060801 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:39:53.850471 | finish at 2026-10-17 19:54:49 + [2026-10-11 21:18:57] iteration 120/ 2384 | consumed samples: 3840 | elapsed time per iteration (ms): 242092.1 | throughput per GPU (TFLOP/s/GPU): 20.2 | MFU 6.48% | FLOPs estimator: workload-aware | logical tokens/s: 69300.9 | valid tokens/s: 69300.9 | physical tokens/s: 70401.0 | loss tokens/s: 49727.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.421905E-01 | lm-global-supervised-tokens: 1.203851E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.272875E-02 | lm-token-buffer-mib: 2.383997E+02 | global_load_balancing_loss: 1.001579E+00 | loss scale: 1.0 | grad norm: 0.193 | params norm: 11228.965 | params rms: 0.060801 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 8:14:56.544952 | finish at 2026-10-18 05:33:54 +times across ranks (ms): + save-checkpoint: + rank 0: 23255.35 + rank 1: 23255.36 + rank 2: 23255.34 + rank 3: 23255.37 + rank 4: 23255.35 + rank 5: 23255.35 + rank 6: 23255.35 + rank 7: 23255.34 + rank 8: 23255.38 + rank 9: 23255.39 + rank 10: 23255.39 + rank 11: 23255.37 + rank 12: 23255.37 + rank 13: 23255.37 + rank 14: 23255.38 + rank 15: 23255.37 + rank 16: 23255.30 + rank 17: 23255.31 + rank 18: 23255.32 + rank 19: 23255.32 + rank 20: 23255.30 + rank 21: 23255.31 + rank 22: 23255.31 + rank 23: 23255.32 + rank 24: 23255.37 + rank 25: 23255.12 + rank 26: 23255.42 + rank 27: 23255.38 + rank 28: 23255.40 + rank 29: 23255.40 + rank 30: 23255.39 + rank 31: 23255.38 + rank 32: 23255.33 + rank 33: 23255.32 + rank 34: 23255.31 + rank 35: 23255.35 + rank 36: 23255.29 + rank 37: 23255.31 + rank 38: 23255.29 + rank 39: 23255.30 + rank 40: 23255.77 + rank 41: 23255.75 + rank 42: 23255.76 + rank 43: 23255.78 + rank 44: 23255.74 + rank 45: 23255.76 + rank 46: 23255.77 + rank 47: 23255.79 + rank 48: 23255.23 + rank 49: 23255.19 + rank 50: 23255.24 + rank 51: 23255.20 + rank 52: 23255.20 + rank 53: 23255.18 + rank 54: 23255.19 + rank 55: 23255.19 + rank 56: 23255.36 + rank 57: 23255.38 + rank 58: 23255.38 + rank 59: 23255.38 + rank 60: 23255.42 + rank 61: 23255.38 + rank 62: 23255.36 + rank 63: 23255.39 + [2026-10-11 21:23:01] iteration 121/ 2384 | consumed samples: 3872 | elapsed time per iteration (ms): 220988.3 | throughput per GPU (TFLOP/s/GPU): 24.9 | MFU 7.98% | FLOPs estimator: workload-aware | logical tokens/s: 75918.9 | valid tokens/s: 75918.9 | physical tokens/s: 76448.7 | loss tokens/s: 59616.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.536566E-01 | lm-global-supervised-tokens: 1.317460E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.737567E-02 | lm-token-buffer-mib: 2.333553E+02 | global_load_balancing_loss: 1.003921E+00 | loss scale: 1.0 | grad norm: 0.114 | params norm: 11228.936 | params rms: 0.060801 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:54:56.441622 | finish at 2026-10-17 16:17:58 + [2026-10-11 21:26:48] iteration 122/ 2384 | consumed samples: 3904 | elapsed time per iteration (ms): 226557.0 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.71% | FLOPs estimator: workload-aware | logical tokens/s: 74052.8 | valid tokens/s: 74052.8 | physical tokens/s: 74487.4 | loss tokens/s: 50482.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.167017E-01 | lm-global-supervised-tokens: 1.143714E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.056727E-02 | lm-token-buffer-mib: 2.327152E+02 | global_load_balancing_loss: 1.000908E+00 | loss scale: 1.0 | grad norm: 0.106 | params norm: 11228.900 | params rms: 0.060801 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:21:12.009326 | finish at 2026-10-17 19:48:00 + [2026-10-11 21:31:38] iteration 123/ 2384 | consumed samples: 3936 | elapsed time per iteration (ms): 290075.4 | throughput per GPU (TFLOP/s/GPU): 17.2 | MFU 5.51% | FLOPs estimator: workload-aware | logical tokens/s: 57837.3 | valid tokens/s: 57837.3 | physical tokens/s: 58574.6 | loss tokens/s: 42608.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.758559E-01 | lm-global-supervised-tokens: 1.235969E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.194585E-02 | lm-token-buffer-mib: 2.369176E+02 | global_load_balancing_loss: 1.000974E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11228.861 | params rms: 0.060801 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 14:11:00.504994 | finish at 2026-10-19 11:42:38 + [2026-10-11 21:35:18] iteration 124/ 2384 | consumed samples: 3968 | elapsed time per iteration (ms): 219724.5 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.06% | FLOPs estimator: workload-aware | logical tokens/s: 76355.6 | valid tokens/s: 76355.6 | physical tokens/s: 76753.9 | loss tokens/s: 56210.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.528232E-01 | lm-global-supervised-tokens: 1.235078E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 5.090060E-02 | lm-token-buffer-mib: 2.336294E+02 | global_load_balancing_loss: 9.998223E-01 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11228.821 | params rms: 0.060800 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:56:17.365556 | finish at 2026-10-17 15:31:35 + [2026-10-11 21:39:03] iteration 125/ 2384 | consumed samples: 4000 | elapsed time per iteration (ms): 225100.5 | throughput per GPU (TFLOP/s/GPU): 19.9 | MFU 6.37% | FLOPs estimator: workload-aware | logical tokens/s: 74532.1 | valid tokens/s: 74532.1 | physical tokens/s: 75682.2 | loss tokens/s: 57388.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.403578E-01 | lm-global-supervised-tokens: 1.291810E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.427496E-02 | lm-token-buffer-mib: 2.362575E+02 | global_load_balancing_loss: 1.000776E+00 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11228.779 | params rms: 0.060800 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:15:01.927410 | finish at 2026-10-17 18:54:05 + [2026-10-11 21:42:45] iteration 126/ 2384 | consumed samples: 4032 | elapsed time per iteration (ms): 222498.5 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.42% | FLOPs estimator: workload-aware | logical tokens/s: 75403.6 | valid tokens/s: 75403.6 | physical tokens/s: 75836.1 | loss tokens/s: 57489.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.903303E-01 | lm-global-supervised-tokens: 1.279142E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.577711E-02 | lm-token-buffer-mib: 2.323981E+02 | global_load_balancing_loss: 9.997349E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11228.738 | params rms: 0.060800 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:33:21.570178 | finish at 2026-10-17 17:16:07 + [2026-10-11 21:46:28] iteration 127/ 2384 | consumed samples: 4064 | elapsed time per iteration (ms): 222965.7 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.17% | FLOPs estimator: workload-aware | logical tokens/s: 75245.6 | valid tokens/s: 75245.6 | physical tokens/s: 75864.3 | loss tokens/s: 55749.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.376373E-01 | lm-global-supervised-tokens: 1.243031E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.808980E-02 | lm-token-buffer-mib: 2.347729E+02 | global_load_balancing_loss: 1.002044E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11228.696 | params rms: 0.060800 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:47:13.634206 | finish at 2026-10-17 17:33:42 + [2026-10-11 21:50:10] iteration 128/ 2384 | consumed samples: 4096 | elapsed time per iteration (ms): 222121.7 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.44% | FLOPs estimator: workload-aware | logical tokens/s: 75531.5 | valid tokens/s: 75531.5 | physical tokens/s: 76645.0 | loss tokens/s: 62568.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.772873E-01 | lm-global-supervised-tokens: 1.389789E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.345177E-02 | lm-token-buffer-mib: 2.362272E+02 | global_load_balancing_loss: 1.001541E+00 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11228.654 | params rms: 0.060800 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:11:46.638142 | finish at 2026-10-17 17:01:57 + [2026-10-11 21:53:54] iteration 129/ 2384 | consumed samples: 4128 | elapsed time per iteration (ms): 224070.5 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.94% | FLOPs estimator: workload-aware | logical tokens/s: 74874.5 | valid tokens/s: 74874.5 | physical tokens/s: 75507.4 | loss tokens/s: 55142.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.787602E-01 | lm-global-supervised-tokens: 1.235578E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.659599E-02 | lm-token-buffer-mib: 2.342467E+02 | global_load_balancing_loss: 9.999542E-01 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11228.608 | params rms: 0.060799 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:21:18.946623 | finish at 2026-10-17 18:15:13 + [2026-10-11 21:58:01] iteration 130/ 2384 | consumed samples: 4160 | elapsed time per iteration (ms): 246344.4 | throughput per GPU (TFLOP/s/GPU): 19.0 | MFU 6.08% | FLOPs estimator: workload-aware | logical tokens/s: 68104.6 | valid tokens/s: 68104.6 | physical tokens/s: 68867.0 | loss tokens/s: 54293.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.577134E-01 | lm-global-supervised-tokens: 1.337492E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.870843E-02 | lm-token-buffer-mib: 2.344940E+02 | global_load_balancing_loss: 1.002691E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11228.556 | params rms: 0.060799 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 10:14:20.360736 | finish at 2026-10-18 08:12:21 + [2026-10-11 22:01:43] iteration 131/ 2384 | consumed samples: 4192 | elapsed time per iteration (ms): 222062.5 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.22% | FLOPs estimator: workload-aware | logical tokens/s: 75551.6 | valid tokens/s: 75551.6 | physical tokens/s: 76278.9 | loss tokens/s: 58287.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.018768E-01 | lm-global-supervised-tokens: 1.294336E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.447701E-02 | lm-token-buffer-mib: 2.352457E+02 | global_load_balancing_loss: 1.001520E+00 | loss scale: 1.0 | grad norm: 0.161 | params norm: 11228.502 | params rms: 0.060799 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:58:26.712052 | finish at 2026-10-17 17:00:10 + [2026-10-11 22:05:23] iteration 132/ 2384 | consumed samples: 4224 | elapsed time per iteration (ms): 219929.4 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.02% | FLOPs estimator: workload-aware | logical tokens/s: 76284.3 | valid tokens/s: 76284.3 | physical tokens/s: 76741.2 | loss tokens/s: 57206.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.751138E-01 | lm-global-supervised-tokens: 1.258133E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.847095E-02 | lm-token-buffer-mib: 2.338000E+02 | global_load_balancing_loss: 1.001266E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11228.447 | params rms: 0.060798 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:34:40.908859 | finish at 2026-10-17 15:40:04 + [2026-10-11 22:09:06] iteration 133/ 2384 | consumed samples: 4256 | elapsed time per iteration (ms): 222904.5 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.41% | FLOPs estimator: workload-aware | logical tokens/s: 75266.3 | valid tokens/s: 75266.3 | physical tokens/s: 75931.6 | loss tokens/s: 58258.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.148952E-01 | lm-global-supervised-tokens: 1.298598E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.598614E-02 | lm-token-buffer-mib: 2.336385E+02 | global_load_balancing_loss: 1.001104E+00 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11228.388 | params rms: 0.060798 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:22:37.996780 | finish at 2026-10-17 17:31:44 + [2026-10-11 22:12:47] iteration 134/ 2384 | consumed samples: 4288 | elapsed time per iteration (ms): 221339.8 | throughput per GPU (TFLOP/s/GPU): 19.9 | MFU 6.39% | FLOPs estimator: workload-aware | logical tokens/s: 75798.4 | valid tokens/s: 75798.4 | physical tokens/s: 76390.9 | loss tokens/s: 58328.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.921655E-01 | lm-global-supervised-tokens: 1.291036E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.425901E-02 | lm-token-buffer-mib: 2.336734E+02 | global_load_balancing_loss: 1.001029E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11228.325 | params rms: 0.060798 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:20:14.660776 | finish at 2026-10-17 16:33:02 + [2026-10-11 22:16:29] iteration 135/ 2384 | consumed samples: 4320 | elapsed time per iteration (ms): 222429.8 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.23% | FLOPs estimator: workload-aware | logical tokens/s: 75426.8 | valid tokens/s: 75426.8 | physical tokens/s: 76394.7 | loss tokens/s: 55663.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.090074E-01 | lm-global-supervised-tokens: 1.238121E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.777442E-02 | lm-token-buffer-mib: 2.339950E+02 | global_load_balancing_loss: 9.998091E-01 | loss scale: 1.0 | grad norm: 0.200 | params norm: 11228.259 | params rms: 0.060797 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:57:24.554322 | finish at 2026-10-17 17:13:54 + [2026-10-11 22:20:17] iteration 136/ 2384 | consumed samples: 4352 | elapsed time per iteration (ms): 227572.1 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.57% | FLOPs estimator: workload-aware | logical tokens/s: 73722.4 | valid tokens/s: 73722.4 | physical tokens/s: 74593.1 | loss tokens/s: 54965.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.206324E-01 | lm-global-supervised-tokens: 1.250870E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.216415E-02 | lm-token-buffer-mib: 2.363593E+02 | global_load_balancing_loss: 1.000884E+00 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11228.188 | params rms: 0.060797 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:06:22.034000 | finish at 2026-10-17 20:26:39 + [2026-10-11 22:24:06] iteration 137/ 2384 | consumed samples: 4384 | elapsed time per iteration (ms): 229434.9 | throughput per GPU (TFLOP/s/GPU): 24.1 | MFU 7.71% | FLOPs estimator: workload-aware | logical tokens/s: 73123.8 | valid tokens/s: 73123.8 | physical tokens/s: 74235.4 | loss tokens/s: 55031.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.649649E-01 | lm-global-supervised-tokens: 1.262608E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.489211E-02 | lm-token-buffer-mib: 2.378616E+02 | global_load_balancing_loss: 1.000284E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11228.116 | params rms: 0.060797 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:12:20.312011 | finish at 2026-10-17 21:36:27 + [2026-10-11 22:27:48] iteration 138/ 2384 | consumed samples: 4416 | elapsed time per iteration (ms): 221821.1 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.34% | FLOPs estimator: workload-aware | logical tokens/s: 75634.0 | valid tokens/s: 75634.0 | physical tokens/s: 76625.1 | loss tokens/s: 57867.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.431332E-01 | lm-global-supervised-tokens: 1.283632E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.529922E-02 | lm-token-buffer-mib: 2.359857E+02 | global_load_balancing_loss: 9.998876E-01 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11228.042 | params rms: 0.060796 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:23:30.089385 | finish at 2026-10-17 16:51:18 + [2026-10-11 22:31:32] iteration 139/ 2384 | consumed samples: 4448 | elapsed time per iteration (ms): 223356.9 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.70% | FLOPs estimator: workload-aware | logical tokens/s: 75113.8 | valid tokens/s: 75113.8 | physical tokens/s: 75733.1 | loss tokens/s: 56544.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.202732E-01 | lm-global-supervised-tokens: 1.262970E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.980568E-02 | lm-token-buffer-mib: 2.339857E+02 | global_load_balancing_loss: 1.000968E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11227.968 | params rms: 0.060796 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:17:16.336793 | finish at 2026-10-17 17:48:48 + [2026-10-11 22:35:34] iteration 140/ 2384 | consumed samples: 4480 | elapsed time per iteration (ms): 242178.9 | throughput per GPU (TFLOP/s/GPU): 18.9 | MFU 6.04% | FLOPs estimator: workload-aware | logical tokens/s: 69276.1 | valid tokens/s: 69276.0 | physical tokens/s: 69956.6 | loss tokens/s: 52730.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.698910E-01 | lm-global-supervised-tokens: 1.277015E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.493832E-02 | lm-token-buffer-mib: 2.331728E+02 | global_load_balancing_loss: 9.993447E-01 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11227.892 | params rms: 0.060795 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 6:57:29.356911 | finish at 2026-10-18 05:33:03 + [2026-10-11 22:39:13] iteration 141/ 2384 | consumed samples: 4512 | elapsed time per iteration (ms): 219240.6 | throughput per GPU (TFLOP/s/GPU): 17.7 | MFU 5.66% | FLOPs estimator: workload-aware | logical tokens/s: 76524.2 | valid tokens/s: 76524.2 | physical tokens/s: 77350.7 | loss tokens/s: 59721.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.773452E-01 | lm-global-supervised-tokens: 1.309334E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.399347E-02 | lm-token-buffer-mib: 2.336782E+02 | global_load_balancing_loss: 9.996161E-01 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11227.816 | params rms: 0.060795 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:35:56.561229 | finish at 2026-10-17 15:15:10 + [2026-10-11 22:42:57] iteration 142/ 2384 | consumed samples: 4544 | elapsed time per iteration (ms): 223515.7 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.57% | FLOPs estimator: workload-aware | logical tokens/s: 75060.4 | valid tokens/s: 75060.4 | physical tokens/s: 75807.6 | loss tokens/s: 56788.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.230051E-01 | lm-global-supervised-tokens: 1.269319E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.434761E-02 | lm-token-buffer-mib: 2.357932E+02 | global_load_balancing_loss: 1.001180E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11227.738 | params rms: 0.060795 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:12:02.261100 | finish at 2026-10-17 17:54:59 + [2026-10-11 22:46:42] iteration 143/ 2384 | consumed samples: 4576 | elapsed time per iteration (ms): 225307.1 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.40% | FLOPs estimator: workload-aware | logical tokens/s: 74463.7 | valid tokens/s: 74463.7 | physical tokens/s: 75337.8 | loss tokens/s: 55655.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.283559E-01 | lm-global-supervised-tokens: 1.253959E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.442212E-02 | lm-token-buffer-mib: 2.345326E+02 | global_load_balancing_loss: 1.000391E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11227.659 | params rms: 0.060794 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:15:13.276947 | finish at 2026-10-17 19:01:55 + [2026-10-11 22:50:27] iteration 144/ 2384 | consumed samples: 4608 | elapsed time per iteration (ms): 225326.1 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.01% | FLOPs estimator: workload-aware | logical tokens/s: 74457.4 | valid tokens/s: 74457.4 | physical tokens/s: 75355.3 | loss tokens/s: 51283.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.630206E-01 | lm-global-supervised-tokens: 1.155545E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.804748E-02 | lm-token-buffer-mib: 2.364951E+02 | global_load_balancing_loss: 1.001469E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11227.582 | params rms: 0.060794 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:12:10.447693 | finish at 2026-10-17 19:02:38 + [2026-10-11 22:54:08] iteration 145/ 2384 | consumed samples: 4640 | elapsed time per iteration (ms): 221118.2 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.35% | FLOPs estimator: workload-aware | logical tokens/s: 75874.4 | valid tokens/s: 75874.4 | physical tokens/s: 76265.8 | loss tokens/s: 54095.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.653474E-01 | lm-global-supervised-tokens: 1.196156E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.382828E-02 | lm-token-buffer-mib: 2.323440E+02 | global_load_balancing_loss: 1.000574E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11227.504 | params rms: 0.060793 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:31:23.538908 | finish at 2026-10-17 16:25:32 + [2026-10-11 22:57:49] iteration 146/ 2384 | consumed samples: 4672 | elapsed time per iteration (ms): 221064.6 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.12% | FLOPs estimator: workload-aware | logical tokens/s: 75892.7 | valid tokens/s: 75892.7 | physical tokens/s: 76495.0 | loss tokens/s: 53083.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.023456E-01 | lm-global-supervised-tokens: 1.173493E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.166739E-02 | lm-token-buffer-mib: 2.339254E+02 | global_load_balancing_loss: 1.000652E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11227.423 | params rms: 0.060793 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:25:42.479802 | finish at 2026-10-17 16:23:32 + [2026-10-11 23:01:30] iteration 147/ 2384 | consumed samples: 4704 | elapsed time per iteration (ms): 220843.2 | throughput per GPU (TFLOP/s/GPU): 24.0 | MFU 7.69% | FLOPs estimator: workload-aware | logical tokens/s: 75968.7 | valid tokens/s: 75968.7 | physical tokens/s: 76720.6 | loss tokens/s: 52950.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.603686E-01 | lm-global-supervised-tokens: 1.169378E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.982428E-02 | lm-token-buffer-mib: 2.340462E+02 | global_load_balancing_loss: 1.000522E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11227.338 | params rms: 0.060792 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:13:46.236729 | finish at 2026-10-17 16:15:16 + [2026-10-11 23:05:17] iteration 148/ 2384 | consumed samples: 4736 | elapsed time per iteration (ms): 226901.1 | throughput per GPU (TFLOP/s/GPU): 19.6 | MFU 6.27% | FLOPs estimator: workload-aware | logical tokens/s: 73940.7 | valid tokens/s: 73940.7 | physical tokens/s: 74184.6 | loss tokens/s: 53287.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.322488E-01 | lm-global-supervised-tokens: 1.209097E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.569242E-02 | lm-token-buffer-mib: 2.329827E+02 | global_load_balancing_loss: 1.001937E+00 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11227.255 | params rms: 0.060792 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:55:50.803979 | finish at 2026-10-17 20:01:08 + [2026-10-11 23:08:59] iteration 149/ 2384 | consumed samples: 4768 | elapsed time per iteration (ms): 222139.7 | throughput per GPU (TFLOP/s/GPU): 25.0 | MFU 8.03% | FLOPs estimator: workload-aware | logical tokens/s: 75525.3 | valid tokens/s: 75525.3 | physical tokens/s: 76075.5 | loss tokens/s: 56346.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.950271E-01 | lm-global-supervised-tokens: 1.251672E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.359681E-02 | lm-token-buffer-mib: 2.326213E+02 | global_load_balancing_loss: 1.001508E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11227.172 | params rms: 0.060791 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:54:42.197918 | finish at 2026-10-17 17:03:41 + [2026-10-11 23:13:01] iteration 150/ 2384 | consumed samples: 4800 | elapsed time per iteration (ms): 241489.4 | throughput per GPU (TFLOP/s/GPU): 17.1 | MFU 5.47% | FLOPs estimator: workload-aware | logical tokens/s: 69473.8 | valid tokens/s: 69473.8 | physical tokens/s: 69984.9 | loss tokens/s: 55445.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.290520E-01 | lm-global-supervised-tokens: 1.338950E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.174157E-02 | lm-token-buffer-mib: 2.338004E+02 | global_load_balancing_loss: 1.000286E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11227.093 | params rms: 0.060791 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 5:51:27.369998 | finish at 2026-10-18 05:04:28 + [2026-10-11 23:16:47] iteration 151/ 2384 | consumed samples: 4832 | elapsed time per iteration (ms): 226345.9 | throughput per GPU (TFLOP/s/GPU): 23.7 | MFU 7.59% | FLOPs estimator: workload-aware | logical tokens/s: 74121.9 | valid tokens/s: 74121.9 | physical tokens/s: 74998.8 | loss tokens/s: 51947.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.466212E-01 | lm-global-supervised-tokens: 1.175807E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.531005E-02 | lm-token-buffer-mib: 2.336354E+02 | global_load_balancing_loss: 1.000913E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11227.016 | params rms: 0.060791 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:23:50.385248 | finish at 2026-10-17 19:40:37 + [2026-10-11 23:20:29] iteration 152/ 2384 | consumed samples: 4864 | elapsed time per iteration (ms): 221782.3 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.73% | FLOPs estimator: workload-aware | logical tokens/s: 75647.2 | valid tokens/s: 75647.2 | physical tokens/s: 76152.5 | loss tokens/s: 56246.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.769485E-01 | lm-global-supervised-tokens: 1.247454E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.361975E-02 | lm-token-buffer-mib: 2.331836E+02 | global_load_balancing_loss: 1.000463E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11226.939 | params rms: 0.060790 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:30:17.998335 | finish at 2026-10-17 16:50:47 + [2026-10-11 23:24:13] iteration 153/ 2384 | consumed samples: 4896 | elapsed time per iteration (ms): 224088.7 | throughput per GPU (TFLOP/s/GPU): 24.1 | MFU 7.73% | FLOPs estimator: workload-aware | logical tokens/s: 74868.6 | valid tokens/s: 74868.6 | physical tokens/s: 75694.6 | loss tokens/s: 58405.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.968359E-01 | lm-global-supervised-tokens: 1.308794E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.139637E-02 | lm-token-buffer-mib: 2.350981E+02 | global_load_balancing_loss: 1.001056E+00 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11226.863 | params rms: 0.060790 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:52:21.787166 | finish at 2026-10-17 18:16:35 + [2026-10-11 23:27:58] iteration 154/ 2384 | consumed samples: 4928 | elapsed time per iteration (ms): 224725.0 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.18% | FLOPs estimator: workload-aware | logical tokens/s: 74656.5 | valid tokens/s: 74656.5 | physical tokens/s: 75337.6 | loss tokens/s: 55736.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.057441E-01 | lm-global-supervised-tokens: 1.252541E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.009839E-02 | lm-token-buffer-mib: 2.351440E+02 | global_load_balancing_loss: 9.999186E-01 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11226.785 | params rms: 0.060789 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:12:16.684923 | finish at 2026-10-17 18:40:14 + [2026-10-11 23:31:39] iteration 155/ 2384 | consumed samples: 4960 | elapsed time per iteration (ms): 221155.6 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.87% | FLOPs estimator: workload-aware | logical tokens/s: 75861.5 | valid tokens/s: 75861.5 | physical tokens/s: 76448.2 | loss tokens/s: 56978.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.887991E-01 | lm-global-supervised-tokens: 1.260106E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.758954E-02 | lm-token-buffer-mib: 2.353700E+02 | global_load_balancing_loss: 1.001660E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11226.709 | params rms: 0.060789 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:55:55.805986 | finish at 2026-10-17 16:27:35 + [2026-10-11 23:35:24] iteration 156/ 2384 | consumed samples: 4992 | elapsed time per iteration (ms): 224641.9 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.95% | FLOPs estimator: workload-aware | logical tokens/s: 74684.1 | valid tokens/s: 74684.1 | physical tokens/s: 75684.3 | loss tokens/s: 55829.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.384887E-01 | lm-global-supervised-tokens: 1.254170E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.545674E-02 | lm-token-buffer-mib: 2.362814E+02 | global_load_balancing_loss: 1.001913E+00 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11226.634 | params rms: 0.060789 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:01:42.209646 | finish at 2026-10-17 18:37:06 + [2026-10-11 23:39:08] iteration 157/ 2384 | consumed samples: 5024 | elapsed time per iteration (ms): 224960.0 | throughput per GPU (TFLOP/s/GPU): 26.2 | MFU 8.41% | FLOPs estimator: workload-aware | logical tokens/s: 74578.4 | valid tokens/s: 74578.4 | physical tokens/s: 75120.3 | loss tokens/s: 57756.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.784125E-01 | lm-global-supervised-tokens: 1.299289E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.707153E-02 | lm-token-buffer-mib: 2.342069E+02 | global_load_balancing_loss: 1.001972E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11226.558 | params rms: 0.060788 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:09:45.959376 | finish at 2026-10-17 18:48:54 + [2026-10-11 23:42:56] iteration 158/ 2384 | consumed samples: 5056 | elapsed time per iteration (ms): 227439.5 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.32% | FLOPs estimator: workload-aware | logical tokens/s: 73765.5 | valid tokens/s: 73765.5 | physical tokens/s: 74537.8 | loss tokens/s: 55250.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.116470E-01 | lm-global-supervised-tokens: 1.256606E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.231850E-02 | lm-token-buffer-mib: 2.337899E+02 | global_load_balancing_loss: 1.000653E+00 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11226.481 | params rms: 0.060788 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:38:00.328800 | finish at 2026-10-17 20:20:56 + [2026-10-11 23:46:39] iteration 159/ 2384 | consumed samples: 5088 | elapsed time per iteration (ms): 223468.7 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.14% | FLOPs estimator: workload-aware | logical tokens/s: 75076.2 | valid tokens/s: 75076.2 | physical tokens/s: 75527.7 | loss tokens/s: 56736.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.556369E-01 | lm-global-supervised-tokens: 1.267871E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.420657E-02 | lm-token-buffer-mib: 2.333925E+02 | global_load_balancing_loss: 9.993512E-01 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11226.408 | params rms: 0.060787 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:06:57.843556 | finish at 2026-10-17 17:53:37 + [2026-10-11 23:50:42] iteration 160/ 2384 | consumed samples: 5120 | elapsed time per iteration (ms): 242862.2 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.27% | FLOPs estimator: workload-aware | logical tokens/s: 69080.9 | valid tokens/s: 69080.9 | physical tokens/s: 69862.6 | loss tokens/s: 53645.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.220375E-01 | lm-global-supervised-tokens: 1.302835E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.916016E-02 | lm-token-buffer-mib: 2.340170E+02 | global_load_balancing_loss: 1.000098E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11226.335 | params rms: 0.060787 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 6:02:05.538151 | finish at 2026-10-18 05:52:48 +times across ranks (ms): + save-checkpoint: + rank 0: 25676.92 + rank 1: 25676.89 + rank 2: 25676.87 + rank 3: 25676.78 + rank 4: 25676.87 + rank 5: 25676.90 + rank 6: 25676.90 + rank 7: 25676.90 + rank 8: 25676.96 + rank 9: 25676.99 + rank 10: 25677.04 + rank 11: 25676.98 + rank 12: 25676.97 + rank 13: 25676.95 + rank 14: 25676.97 + rank 15: 25676.97 + rank 16: 25676.60 + rank 17: 25676.59 + rank 18: 25676.62 + rank 19: 25676.60 + rank 20: 25676.61 + rank 21: 25676.59 + rank 22: 25676.62 + rank 23: 25676.60 + rank 24: 25677.16 + rank 25: 25677.11 + rank 26: 25677.09 + rank 27: 25677.11 + rank 28: 25677.06 + rank 29: 25677.12 + rank 30: 25677.11 + rank 31: 25676.65 + rank 32: 25676.58 + rank 33: 25676.52 + rank 34: 25676.57 + rank 35: 25676.61 + rank 36: 25676.54 + rank 37: 25676.54 + rank 38: 25676.51 + rank 39: 25676.51 + rank 40: 25676.81 + rank 41: 25676.80 + rank 42: 25676.78 + rank 43: 25676.80 + rank 44: 25676.78 + rank 45: 25676.77 + rank 46: 25676.79 + rank 47: 25676.81 + rank 48: 25676.99 + rank 49: 25677.01 + rank 50: 25677.13 + rank 51: 25677.09 + rank 52: 25677.04 + rank 53: 25676.89 + rank 54: 25677.07 + rank 55: 25676.84 + rank 56: 25676.61 + rank 57: 25676.66 + rank 58: 25676.66 + rank 59: 25676.65 + rank 60: 25676.64 + rank 61: 25676.56 + rank 62: 25676.60 + rank 63: 25676.58 + [2026-10-11 23:54:54] iteration 161/ 2384 | consumed samples: 5152 | elapsed time per iteration (ms): 225944.5 | throughput per GPU (TFLOP/s/GPU): 19.4 | MFU 6.23% | FLOPs estimator: workload-aware | logical tokens/s: 74253.6 | valid tokens/s: 74253.6 | physical tokens/s: 74935.8 | loss tokens/s: 58479.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.852767E-01 | lm-global-supervised-tokens: 1.321301E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 2.837313E-01 | lm-token-buffer-mib: 2.329227E+02 | global_load_balancing_loss: 9.994587E-01 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11226.262 | params rms: 0.060787 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:31:14.632972 | finish at 2026-10-17 19:26:09 + [2026-10-11 23:58:44] iteration 162/ 2384 | consumed samples: 5184 | elapsed time per iteration (ms): 230281.7 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.76% | FLOPs estimator: workload-aware | logical tokens/s: 72855.0 | valid tokens/s: 72855.0 | physical tokens/s: 73754.3 | loss tokens/s: 53799.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.031324E-01 | lm-global-supervised-tokens: 1.238893E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.178726E-02 | lm-token-buffer-mib: 2.360675E+02 | global_load_balancing_loss: 1.000865E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11226.189 | params rms: 0.060786 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:08:05.980080 | finish at 2026-10-17 22:06:50 + [2026-10-12 00:02:26] iteration 163/ 2384 | consumed samples: 5216 | elapsed time per iteration (ms): 222084.0 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.31% | FLOPs estimator: workload-aware | logical tokens/s: 75544.4 | valid tokens/s: 75544.4 | physical tokens/s: 76126.6 | loss tokens/s: 58377.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.447223E-01 | lm-global-supervised-tokens: 1.296461E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.686892E-02 | lm-token-buffer-mib: 2.324132E+02 | global_load_balancing_loss: 1.001694E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11226.114 | params rms: 0.060786 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:00:48.655854 | finish at 2026-10-17 17:03:15 + [2026-10-12 00:06:07] iteration 164/ 2384 | consumed samples: 5248 | elapsed time per iteration (ms): 220642.6 | throughput per GPU (TFLOP/s/GPU): 23.5 | MFU 7.52% | FLOPs estimator: workload-aware | logical tokens/s: 76037.8 | valid tokens/s: 76037.8 | physical tokens/s: 76569.1 | loss tokens/s: 61253.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.297733E-01 | lm-global-supervised-tokens: 1.351514E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.632008E-02 | lm-token-buffer-mib: 2.335800E+02 | global_load_balancing_loss: 9.998623E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11226.042 | params rms: 0.060785 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:03:46.664758 | finish at 2026-10-17 16:09:54 + [2026-10-12 00:09:49] iteration 165/ 2384 | consumed samples: 5280 | elapsed time per iteration (ms): 222130.0 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.17% | FLOPs estimator: workload-aware | logical tokens/s: 75528.7 | valid tokens/s: 75528.7 | physical tokens/s: 76456.8 | loss tokens/s: 58869.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.376346E-01 | lm-global-supervised-tokens: 1.307663E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.287194E-02 | lm-token-buffer-mib: 2.357268E+02 | global_load_balancing_loss: 1.000108E+00 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11225.970 | params rms: 0.060785 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:55:06.513107 | finish at 2026-10-17 17:04:56 + [2026-10-12 00:13:34] iteration 166/ 2384 | consumed samples: 5312 | elapsed time per iteration (ms): 224806.2 | throughput per GPU (TFLOP/s/GPU): 23.2 | MFU 7.45% | FLOPs estimator: workload-aware | logical tokens/s: 74629.5 | valid tokens/s: 74629.5 | physical tokens/s: 75197.6 | loss tokens/s: 57829.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.055957E-01 | lm-global-supervised-tokens: 1.300043E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.855336E-02 | lm-token-buffer-mib: 2.332480E+02 | global_load_balancing_loss: 1.001221E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11225.894 | params rms: 0.060785 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:30:20.217762 | finish at 2026-10-17 18:43:54 + [2026-10-12 00:17:14] iteration 167/ 2384 | consumed samples: 5344 | elapsed time per iteration (ms): 220506.1 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.71% | FLOPs estimator: workload-aware | logical tokens/s: 76084.9 | valid tokens/s: 76084.9 | physical tokens/s: 76883.2 | loss tokens/s: 56254.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.531193E-01 | lm-global-supervised-tokens: 1.240451E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.623668E-02 | lm-token-buffer-mib: 2.332529E+02 | global_load_balancing_loss: 1.000437E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11225.821 | params rms: 0.060784 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:47:42.008765 | finish at 2026-10-17 16:04:56 + [2026-10-12 00:20:56] iteration 168/ 2384 | consumed samples: 5376 | elapsed time per iteration (ms): 221616.7 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.15% | FLOPs estimator: workload-aware | logical tokens/s: 75703.6 | valid tokens/s: 75703.6 | physical tokens/s: 76615.4 | loss tokens/s: 53906.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.110702E-01 | lm-global-supervised-tokens: 1.194655E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.955727E-02 | lm-token-buffer-mib: 2.344926E+02 | global_load_balancing_loss: 1.000183E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11225.748 | params rms: 0.060784 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:25:02.702154 | finish at 2026-10-17 16:45:59 + [2026-10-12 00:24:41] iteration 169/ 2384 | consumed samples: 5408 | elapsed time per iteration (ms): 224610.8 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.71% | FLOPs estimator: workload-aware | logical tokens/s: 74694.6 | valid tokens/s: 74694.6 | physical tokens/s: 75937.8 | loss tokens/s: 56470.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.103942E-01 | lm-global-supervised-tokens: 1.268387E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.580772E-02 | lm-token-buffer-mib: 2.370105E+02 | global_load_balancing_loss: 9.996024E-01 | loss scale: 1.0 | grad norm: 0.130 | params norm: 11225.675 | params rms: 0.060783 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:11:52.988074 | finish at 2026-10-17 18:36:34 + [2026-10-12 00:28:46] iteration 170/ 2384 | consumed samples: 5440 | elapsed time per iteration (ms): 245072.9 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.85% | FLOPs estimator: workload-aware | logical tokens/s: 68458.0 | valid tokens/s: 68458.0 | physical tokens/s: 69328.7 | loss tokens/s: 49678.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.090848E-01 | lm-global-supervised-tokens: 1.217486E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.415072E-02 | lm-token-buffer-mib: 2.354704E+02 | global_load_balancing_loss: 1.000550E+00 | loss scale: 1.0 | grad norm: 0.133 | params norm: 11225.601 | params rms: 0.060783 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 6:43:11.492573 | finish at 2026-10-18 07:11:57 + [2026-10-12 00:32:27] iteration 171/ 2384 | consumed samples: 5472 | elapsed time per iteration (ms): 221183.5 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.34% | FLOPs estimator: workload-aware | logical tokens/s: 75851.9 | valid tokens/s: 75851.9 | physical tokens/s: 76220.9 | loss tokens/s: 58616.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.797973E-01 | lm-global-supervised-tokens: 1.296495E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.100397E-02 | lm-token-buffer-mib: 2.334531E+02 | global_load_balancing_loss: 9.996896E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11225.525 | params rms: 0.060783 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:57:59.156315 | finish at 2026-10-17 16:30:26 + [2026-10-12 00:36:08] iteration 172/ 2384 | consumed samples: 5504 | elapsed time per iteration (ms): 221341.5 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.16% | FLOPs estimator: workload-aware | logical tokens/s: 75797.8 | valid tokens/s: 75797.8 | physical tokens/s: 76086.2 | loss tokens/s: 56233.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.855377E-01 | lm-global-supervised-tokens: 1.244684E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.198069E-02 | lm-token-buffer-mib: 2.333381E+02 | global_load_balancing_loss: 1.000795E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11225.450 | params rms: 0.060782 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:00:07.329537 | finish at 2026-10-17 16:36:15 + [2026-10-12 00:39:48] iteration 173/ 2384 | consumed samples: 5536 | elapsed time per iteration (ms): 219837.7 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.90% | FLOPs estimator: workload-aware | logical tokens/s: 76316.2 | valid tokens/s: 76316.2 | physical tokens/s: 76864.3 | loss tokens/s: 58163.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.566285E-01 | lm-global-supervised-tokens: 1.278647E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.374965E-02 | lm-token-buffer-mib: 2.342867E+02 | global_load_balancing_loss: 1.001073E+00 | loss scale: 1.0 | grad norm: 0.107 | params norm: 11225.374 | params rms: 0.060782 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:01:01.221404 | finish at 2026-10-17 15:40:49 + [2026-10-12 00:43:31] iteration 174/ 2384 | consumed samples: 5568 | elapsed time per iteration (ms): 222829.0 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.19% | FLOPs estimator: workload-aware | logical tokens/s: 75291.7 | valid tokens/s: 75291.7 | physical tokens/s: 76388.2 | loss tokens/s: 60420.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.714797E-01 | lm-global-supervised-tokens: 1.346344E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.033878E-02 | lm-token-buffer-mib: 2.335940E+02 | global_load_balancing_loss: 1.002322E+00 | loss scale: 1.0 | grad norm: 0.161 | params norm: 11225.293 | params rms: 0.060781 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:47:32.125821 | finish at 2026-10-17 17:31:03 + [2026-10-12 00:47:15] iteration 175/ 2384 | consumed samples: 5600 | elapsed time per iteration (ms): 224477.0 | throughput per GPU (TFLOP/s/GPU): 18.3 | MFU 5.86% | FLOPs estimator: workload-aware | logical tokens/s: 74739.0 | valid tokens/s: 74739.0 | physical tokens/s: 75742.4 | loss tokens/s: 58614.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.167377E-01 | lm-global-supervised-tokens: 1.315760E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.494174E-02 | lm-token-buffer-mib: 2.330754E+02 | global_load_balancing_loss: 1.000387E+00 | loss scale: 1.0 | grad norm: 0.139 | params norm: 11225.216 | params rms: 0.060781 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:44:29.588716 | finish at 2026-10-17 18:31:45 + [2026-10-12 00:51:01] iteration 176/ 2384 | consumed samples: 5632 | elapsed time per iteration (ms): 226062.5 | throughput per GPU (TFLOP/s/GPU): 24.1 | MFU 7.71% | FLOPs estimator: workload-aware | logical tokens/s: 74214.7 | valid tokens/s: 74214.7 | physical tokens/s: 74907.4 | loss tokens/s: 53621.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.407216E-01 | lm-global-supervised-tokens: 1.212184E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.252905E-02 | lm-token-buffer-mib: 2.332556E+02 | global_load_balancing_loss: 1.002494E+00 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11225.142 | params rms: 0.060780 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:39:05.984207 | finish at 2026-10-17 19:30:07 + [2026-10-12 00:54:43] iteration 177/ 2384 | consumed samples: 5664 | elapsed time per iteration (ms): 221806.6 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.98% | FLOPs estimator: workload-aware | logical tokens/s: 75638.8 | valid tokens/s: 75638.8 | physical tokens/s: 76414.0 | loss tokens/s: 57666.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.723358E-01 | lm-global-supervised-tokens: 1.279072E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.796028E-02 | lm-token-buffer-mib: 2.333643E+02 | global_load_balancing_loss: 1.000134E+00 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11225.072 | params rms: 0.060780 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:58:47.166407 | finish at 2026-10-17 16:53:30 + [2026-10-12 00:58:23] iteration 178/ 2384 | consumed samples: 5696 | elapsed time per iteration (ms): 219655.9 | throughput per GPU (TFLOP/s/GPU): 19.7 | MFU 6.33% | FLOPs estimator: workload-aware | logical tokens/s: 76379.3 | valid tokens/s: 76379.3 | physical tokens/s: 77042.1 | loss tokens/s: 55383.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.205460E-01 | lm-global-supervised-tokens: 1.216534E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.357751E-02 | lm-token-buffer-mib: 2.327698E+02 | global_load_balancing_loss: 9.992905E-01 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11225.005 | params rms: 0.060780 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:36:00.831777 | finish at 2026-10-17 15:34:24 + [2026-10-12 01:02:05] iteration 179/ 2384 | consumed samples: 5728 | elapsed time per iteration (ms): 222556.8 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.42% | FLOPs estimator: workload-aware | logical tokens/s: 75383.9 | valid tokens/s: 75383.9 | physical tokens/s: 76026.7 | loss tokens/s: 57233.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.987313E-01 | lm-global-supervised-tokens: 1.273778E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.269511E-02 | lm-token-buffer-mib: 2.337025E+02 | global_load_balancing_loss: 9.992995E-01 | loss scale: 1.0 | grad norm: 0.176 | params norm: 11224.941 | params rms: 0.060779 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:18:57.780554 | finish at 2026-10-17 17:21:03 + [2026-10-12 01:06:09] iteration 180/ 2384 | consumed samples: 5760 | elapsed time per iteration (ms): 243405.4 | throughput per GPU (TFLOP/s/GPU): 19.2 | MFU 6.16% | FLOPs estimator: workload-aware | logical tokens/s: 68926.9 | valid tokens/s: 68926.9 | physical tokens/s: 69742.9 | loss tokens/s: 53136.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.126995E-01 | lm-global-supervised-tokens: 1.293370E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.745189E-02 | lm-token-buffer-mib: 2.347305E+02 | global_load_balancing_loss: 9.987878E-01 | loss scale: 1.0 | grad norm: 0.124 | params norm: 11224.875 | params rms: 0.060779 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 5:01:05.424438 | finish at 2026-10-18 06:07:14 + [2026-10-12 01:09:53] iteration 181/ 2384 | consumed samples: 5792 | elapsed time per iteration (ms): 224171.4 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.29% | FLOPs estimator: workload-aware | logical tokens/s: 74840.8 | valid tokens/s: 74840.8 | physical tokens/s: 76036.1 | loss tokens/s: 58760.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.508860E-01 | lm-global-supervised-tokens: 1.317241E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.490596E-02 | lm-token-buffer-mib: 2.371384E+02 | global_load_balancing_loss: 1.001397E+00 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11224.809 | params rms: 0.060779 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:10:49.518195 | finish at 2026-10-17 18:20:42 + [2026-10-12 01:13:33] iteration 182/ 2384 | consumed samples: 5824 | elapsed time per iteration (ms): 220237.0 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.78% | FLOPs estimator: workload-aware | logical tokens/s: 76178.0 | valid tokens/s: 76178.0 | physical tokens/s: 76859.2 | loss tokens/s: 58780.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.634713E-01 | lm-global-supervised-tokens: 1.294564E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.402731E-02 | lm-token-buffer-mib: 2.352774E+02 | global_load_balancing_loss: 1.001024E+00 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11224.743 | params rms: 0.060778 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:42:41.909675 | finish at 2026-10-17 15:56:15 + [2026-10-12 01:17:12] iteration 183/ 2384 | consumed samples: 5856 | elapsed time per iteration (ms): 219213.6 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.89% | FLOPs estimator: workload-aware | logical tokens/s: 76533.6 | valid tokens/s: 76533.6 | physical tokens/s: 77160.2 | loss tokens/s: 56586.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.017923E-01 | lm-global-supervised-tokens: 1.240448E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.831781E-02 | lm-token-buffer-mib: 2.348916E+02 | global_load_balancing_loss: 1.000229E+00 | loss scale: 1.0 | grad norm: 0.187 | params norm: 11224.677 | params rms: 0.060778 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:01:29.141821 | finish at 2026-10-17 15:18:42 + [2026-10-12 01:20:55] iteration 184/ 2384 | consumed samples: 5888 | elapsed time per iteration (ms): 222229.9 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.29% | FLOPs estimator: workload-aware | logical tokens/s: 75494.8 | valid tokens/s: 75494.8 | physical tokens/s: 76258.9 | loss tokens/s: 56748.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.072042E-01 | lm-global-supervised-tokens: 1.261111E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.357504E-02 | lm-token-buffer-mib: 2.342166E+02 | global_load_balancing_loss: 1.003136E+00 | loss scale: 1.0 | grad norm: 0.279 | params norm: 11224.610 | params rms: 0.060778 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:48:25.749321 | finish at 2026-10-17 17:09:20 + [2026-10-12 01:24:43] iteration 185/ 2384 | consumed samples: 5920 | elapsed time per iteration (ms): 228329.5 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.88% | FLOPs estimator: workload-aware | logical tokens/s: 73478.0 | valid tokens/s: 73478.0 | physical tokens/s: 74724.8 | loss tokens/s: 53840.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.912447E-01 | lm-global-supervised-tokens: 1.229338E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.497845E-02 | lm-token-buffer-mib: 2.391257E+02 | global_load_balancing_loss: 1.002072E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11224.543 | params rms: 0.060777 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:28:16.632277 | finish at 2026-10-17 20:53:00 + [2026-10-12 01:28:25] iteration 186/ 2384 | consumed samples: 5952 | elapsed time per iteration (ms): 222154.9 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.96% | FLOPs estimator: workload-aware | logical tokens/s: 75520.1 | valid tokens/s: 75520.1 | physical tokens/s: 76417.2 | loss tokens/s: 63312.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.822520E-01 | lm-global-supervised-tokens: 1.406520E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.032479E-02 | lm-token-buffer-mib: 2.337745E+02 | global_load_balancing_loss: 1.001919E+00 | loss scale: 1.0 | grad norm: 0.211 | params norm: 11224.478 | params rms: 0.060777 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:38:16.446781 | finish at 2026-10-17 17:06:42 + [2026-10-12 01:32:06] iteration 187/ 2384 | consumed samples: 5984 | elapsed time per iteration (ms): 221052.1 | throughput per GPU (TFLOP/s/GPU): 24.2 | MFU 7.77% | FLOPs estimator: workload-aware | logical tokens/s: 75897.0 | valid tokens/s: 75897.0 | physical tokens/s: 76285.7 | loss tokens/s: 58176.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.245247E-01 | lm-global-supervised-tokens: 1.286004E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.137442E-02 | lm-token-buffer-mib: 2.334988E+02 | global_load_balancing_loss: 9.986920E-01 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11224.412 | params rms: 0.060777 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:54:11.471956 | finish at 2026-10-17 16:26:18 + [2026-10-12 01:35:50] iteration 188/ 2384 | consumed samples: 6016 | elapsed time per iteration (ms): 223415.1 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.34% | FLOPs estimator: workload-aware | logical tokens/s: 75094.2 | valid tokens/s: 75094.2 | physical tokens/s: 75503.4 | loss tokens/s: 55009.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.120548E-01 | lm-global-supervised-tokens: 1.228994E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.344971E-02 | lm-token-buffer-mib: 2.334737E+02 | global_load_balancing_loss: 9.975771E-01 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11224.343 | params rms: 0.060776 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:16:59.647250 | finish at 2026-10-17 17:52:49 + [2026-10-12 01:39:32] iteration 189/ 2384 | consumed samples: 6048 | elapsed time per iteration (ms): 222243.2 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.82% | FLOPs estimator: workload-aware | logical tokens/s: 75490.3 | valid tokens/s: 75490.3 | physical tokens/s: 75899.0 | loss tokens/s: 55300.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.026169E-01 | lm-global-supervised-tokens: 1.229026E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.642649E-02 | lm-token-buffer-mib: 2.331660E+02 | global_load_balancing_loss: 9.999895E-01 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11224.270 | params rms: 0.060776 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:30:23.840363 | finish at 2026-10-17 17:09:56 + [2026-10-12 01:43:37] iteration 190/ 2384 | consumed samples: 6080 | elapsed time per iteration (ms): 245558.1 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.60% | FLOPs estimator: workload-aware | logical tokens/s: 68322.8 | valid tokens/s: 68322.8 | physical tokens/s: 68825.0 | loss tokens/s: 54455.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.081380E-01 | lm-global-supervised-tokens: 1.337192E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.382632E-02 | lm-token-buffer-mib: 2.331953E+02 | global_load_balancing_loss: 1.001040E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11224.199 | params rms: 0.060775 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 5:39:14.375642 | finish at 2026-10-18 07:22:52 + [2026-10-12 01:47:22] iteration 191/ 2384 | consumed samples: 6112 | elapsed time per iteration (ms): 224432.2 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.19% | FLOPs estimator: workload-aware | logical tokens/s: 74753.9 | valid tokens/s: 74753.9 | physical tokens/s: 75379.1 | loss tokens/s: 59878.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.093939E-01 | lm-global-supervised-tokens: 1.343873E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.766610E-02 | lm-token-buffer-mib: 2.340310E+02 | global_load_balancing_loss: 1.000216E+00 | loss scale: 1.0 | grad norm: 0.159 | params norm: 11224.127 | params rms: 0.060775 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:42:59.822344 | finish at 2026-10-17 18:30:22 + [2026-10-12 01:51:04] iteration 192/ 2384 | consumed samples: 6144 | elapsed time per iteration (ms): 222607.8 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.46% | FLOPs estimator: workload-aware | logical tokens/s: 75366.5 | valid tokens/s: 75366.5 | physical tokens/s: 76256.8 | loss tokens/s: 55992.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.910332E-01 | lm-global-supervised-tokens: 1.246441E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.055388E-02 | lm-token-buffer-mib: 2.339784E+02 | global_load_balancing_loss: 1.001927E+00 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11224.055 | params rms: 0.060775 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:32:36.321133 | finish at 2026-10-17 17:23:41 + [2026-10-12 01:54:47] iteration 193/ 2384 | consumed samples: 6176 | elapsed time per iteration (ms): 222935.6 | throughput per GPU (TFLOP/s/GPU): 24.9 | MFU 7.97% | FLOPs estimator: workload-aware | logical tokens/s: 75255.6 | valid tokens/s: 75255.6 | physical tokens/s: 75822.9 | loss tokens/s: 49270.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.182750E-01 | lm-global-supervised-tokens: 1.098418E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.330302E-02 | lm-token-buffer-mib: 2.344652E+02 | global_load_balancing_loss: 1.001835E+00 | loss scale: 1.0 | grad norm: 0.154 | params norm: 11223.981 | params rms: 0.060774 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:40:51.796785 | finish at 2026-10-17 17:35:39 + [2026-10-12 01:58:27] iteration 194/ 2384 | consumed samples: 6208 | elapsed time per iteration (ms): 219266.7 | throughput per GPU (TFLOP/s/GPU): 19.9 | MFU 6.38% | FLOPs estimator: workload-aware | logical tokens/s: 76515.1 | valid tokens/s: 76515.1 | physical tokens/s: 77178.6 | loss tokens/s: 60904.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.552430E-01 | lm-global-supervised-tokens: 1.335432E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.637481E-02 | lm-token-buffer-mib: 2.334655E+02 | global_load_balancing_loss: 1.001672E+00 | loss scale: 1.0 | grad norm: 0.256 | params norm: 11223.903 | params rms: 0.060774 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:23:14.019806 | finish at 2026-10-17 15:21:41 + [2026-10-12 02:02:09] iteration 195/ 2384 | consumed samples: 6240 | elapsed time per iteration (ms): 221889.4 | throughput per GPU (TFLOP/s/GPU): 19.5 | MFU 6.24% | FLOPs estimator: workload-aware | logical tokens/s: 75610.6 | valid tokens/s: 75610.6 | physical tokens/s: 76665.8 | loss tokens/s: 63399.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.334089E-01 | lm-global-supervised-tokens: 1.406761E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.845316E-02 | lm-token-buffer-mib: 2.348499E+02 | global_load_balancing_loss: 1.001364E+00 | loss scale: 1.0 | grad norm: 0.547 | params norm: 11223.830 | params rms: 0.060773 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:55:15.888783 | finish at 2026-10-17 16:57:24 + [2026-10-12 02:05:52] iteration 196/ 2384 | consumed samples: 6272 | elapsed time per iteration (ms): 223378.7 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.89% | FLOPs estimator: workload-aware | logical tokens/s: 75106.6 | valid tokens/s: 75106.6 | physical tokens/s: 76303.1 | loss tokens/s: 59090.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.547132E-01 | lm-global-supervised-tokens: 1.319950E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.577043E-02 | lm-token-buffer-mib: 2.360246E+02 | global_load_balancing_loss: 1.000656E+00 | loss scale: 1.0 | grad norm: 0.340 | params norm: 11223.761 | params rms: 0.060773 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:45:52.577381 | finish at 2026-10-17 17:51:44 + [2026-10-12 02:09:36] iteration 197/ 2384 | consumed samples: 6304 | elapsed time per iteration (ms): 223684.2 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.81% | FLOPs estimator: workload-aware | logical tokens/s: 75004.0 | valid tokens/s: 75004.0 | physical tokens/s: 75441.2 | loss tokens/s: 56100.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.708198E-01 | lm-global-supervised-tokens: 1.254877E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.284621E-02 | lm-token-buffer-mib: 2.332295E+02 | global_load_balancing_loss: 1.000383E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11223.693 | params rms: 0.060773 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:53:17.336642 | finish at 2026-10-17 18:02:53 + [2026-10-12 02:13:16] iteration 198/ 2384 | consumed samples: 6336 | elapsed time per iteration (ms): 220040.6 | throughput per GPU (TFLOP/s/GPU): 24.9 | MFU 7.98% | FLOPs estimator: workload-aware | logical tokens/s: 76245.8 | valid tokens/s: 76245.8 | physical tokens/s: 77008.1 | loss tokens/s: 58230.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.174424E-01 | lm-global-supervised-tokens: 1.281307E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.481799E-02 | lm-token-buffer-mib: 2.329339E+02 | global_load_balancing_loss: 1.001467E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11223.626 | params rms: 0.060772 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:36:48.645413 | finish at 2026-10-17 15:50:04 + [2026-10-12 02:17:01] iteration 199/ 2384 | consumed samples: 6368 | elapsed time per iteration (ms): 225362.7 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.02% | FLOPs estimator: workload-aware | logical tokens/s: 74445.4 | valid tokens/s: 74445.4 | physical tokens/s: 75254.5 | loss tokens/s: 57743.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.725706E-01 | lm-global-supervised-tokens: 1.301315E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.621228E-02 | lm-token-buffer-mib: 2.363530E+02 | global_load_balancing_loss: 1.002725E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11223.559 | params rms: 0.060772 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:46:57.449979 | finish at 2026-10-17 19:03:58 + [2026-10-12 02:21:01] iteration 200/ 2384 | consumed samples: 6400 | elapsed time per iteration (ms): 240485.8 | throughput per GPU (TFLOP/s/GPU): 18.8 | MFU 6.02% | FLOPs estimator: workload-aware | logical tokens/s: 69763.8 | valid tokens/s: 69763.8 | physical tokens/s: 70767.2 | loss tokens/s: 53941.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.648332E-01 | lm-global-supervised-tokens: 1.297222E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.832846E-02 | lm-token-buffer-mib: 2.345135E+02 | global_load_balancing_loss: 1.000237E+00 | loss scale: 1.0 | grad norm: 0.248 | params norm: 11223.493 | params rms: 0.060772 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:53:40.955498 | finish at 2026-10-18 04:14:42 +times across ranks (ms): + save-checkpoint: + rank 0: 23273.12 + rank 1: 23273.14 + rank 2: 23273.13 + rank 3: 23273.13 + rank 4: 23273.12 + rank 5: 23273.16 + rank 6: 23273.12 + rank 7: 23273.15 + rank 8: 23273.57 + rank 9: 23273.56 + rank 10: 23273.57 + rank 11: 23273.55 + rank 12: 23273.54 + rank 13: 23273.59 + rank 14: 23273.55 + rank 15: 23273.55 + rank 16: 23273.33 + rank 17: 23273.35 + rank 18: 23273.33 + rank 19: 23273.35 + rank 20: 23273.31 + rank 21: 23273.30 + rank 22: 23273.32 + rank 23: 23273.30 + rank 24: 23273.86 + rank 25: 23273.86 + rank 26: 23273.83 + rank 27: 23273.85 + rank 28: 23273.81 + rank 29: 23273.87 + rank 30: 23274.08 + rank 31: 23273.84 + rank 32: 23272.28 + rank 33: 23272.22 + rank 34: 23272.24 + rank 35: 23272.26 + rank 36: 23272.22 + rank 37: 23272.25 + rank 38: 23272.20 + rank 39: 23272.22 + rank 40: 23272.92 + rank 41: 23272.88 + rank 42: 23272.90 + rank 43: 23272.89 + rank 44: 23272.88 + rank 45: 23272.88 + rank 46: 23272.88 + rank 47: 23272.87 + rank 48: 23273.17 + rank 49: 23273.16 + rank 50: 23273.20 + rank 51: 23273.16 + rank 52: 23273.15 + rank 53: 23273.13 + rank 54: 23273.14 + rank 55: 23273.15 + rank 56: 23273.00 + rank 57: 23273.02 + rank 58: 23273.02 + rank 59: 23273.01 + rank 60: 23273.01 + rank 61: 23273.00 + rank 62: 23273.02 + rank 63: 23273.01 + [2026-10-12 02:25:05] iteration 201/ 2384 | consumed samples: 6432 | elapsed time per iteration (ms): 220002.6 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.93% | FLOPs estimator: workload-aware | logical tokens/s: 76259.2 | valid tokens/s: 76259.2 | physical tokens/s: 76881.7 | loss tokens/s: 57868.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.579491E-01 | lm-global-supervised-tokens: 1.273129E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.767904E-02 | lm-token-buffer-mib: 2.342970E+02 | global_load_balancing_loss: 9.997268E-01 | loss scale: 1.0 | grad norm: 0.113 | params norm: 11223.424 | params rms: 0.060771 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:24:25.674660 | finish at 2026-10-17 15:49:30 + [2026-10-12 02:28:51] iteration 202/ 2384 | consumed samples: 6464 | elapsed time per iteration (ms): 226437.4 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.32% | FLOPs estimator: workload-aware | logical tokens/s: 74091.9 | valid tokens/s: 74091.9 | physical tokens/s: 74399.0 | loss tokens/s: 55807.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.217194E-01 | lm-global-supervised-tokens: 1.263683E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.212635E-02 | lm-token-buffer-mib: 2.331131E+02 | global_load_balancing_loss: 9.996564E-01 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11223.357 | params rms: 0.060771 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:14:46.401822 | finish at 2026-10-17 19:43:38 + [2026-10-12 02:32:31] iteration 203/ 2384 | consumed samples: 6496 | elapsed time per iteration (ms): 219445.9 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.81% | FLOPs estimator: workload-aware | logical tokens/s: 76452.5 | valid tokens/s: 76452.5 | physical tokens/s: 77356.7 | loss tokens/s: 58999.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.917669E-01 | lm-global-supervised-tokens: 1.294710E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.946903E-02 | lm-token-buffer-mib: 2.341862E+02 | global_load_balancing_loss: 9.999141E-01 | loss scale: 1.0 | grad norm: 0.354 | params norm: 11223.291 | params rms: 0.060770 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:56:51.468301 | finish at 2026-10-17 15:29:22 + [2026-10-12 02:36:14] iteration 204/ 2384 | consumed samples: 6528 | elapsed time per iteration (ms): 223120.5 | throughput per GPU (TFLOP/s/GPU): 18.9 | MFU 6.07% | FLOPs estimator: workload-aware | logical tokens/s: 75193.5 | valid tokens/s: 75193.5 | physical tokens/s: 75902.3 | loss tokens/s: 59433.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.542060E-01 | lm-global-supervised-tokens: 1.326076E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.323533E-02 | lm-token-buffer-mib: 2.339587E+02 | global_load_balancing_loss: 1.000012E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11223.225 | params rms: 0.060770 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:06:42.654848 | finish at 2026-10-17 17:42:56 + [2026-10-12 02:39:54] iteration 205/ 2384 | consumed samples: 6560 | elapsed time per iteration (ms): 220262.5 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.40% | FLOPs estimator: workload-aware | logical tokens/s: 76169.0 | valid tokens/s: 76169.0 | physical tokens/s: 76610.6 | loss tokens/s: 51806.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.431423E-01 | lm-global-supervised-tokens: 1.141098E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.629769E-02 | lm-token-buffer-mib: 2.334006E+02 | global_load_balancing_loss: 1.001105E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11223.159 | params rms: 0.060770 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:19:11.951757 | finish at 2026-10-17 15:59:06 + [2026-10-12 02:43:35] iteration 206/ 2384 | consumed samples: 6592 | elapsed time per iteration (ms): 221322.4 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.89% | FLOPs estimator: workload-aware | logical tokens/s: 75804.2 | valid tokens/s: 75804.2 | physical tokens/s: 76315.4 | loss tokens/s: 58102.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.641169E-01 | lm-global-supervised-tokens: 1.285943E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.559403E-02 | lm-token-buffer-mib: 2.326353E+02 | global_load_balancing_loss: 1.001847E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11223.091 | params rms: 0.060769 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:54:00.236215 | finish at 2026-10-17 16:37:36 + [2026-10-12 02:47:16] iteration 207/ 2384 | consumed samples: 6624 | elapsed time per iteration (ms): 220692.0 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.64% | FLOPs estimator: workload-aware | logical tokens/s: 76020.9 | valid tokens/s: 76020.9 | physical tokens/s: 76737.3 | loss tokens/s: 56188.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.721605E-01 | lm-global-supervised-tokens: 1.240028E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.090949E-02 | lm-token-buffer-mib: 2.341474E+02 | global_load_balancing_loss: 1.000909E+00 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11223.022 | params rms: 0.060769 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:27:26.386230 | finish at 2026-10-17 16:14:42 + [2026-10-12 02:50:59] iteration 208/ 2384 | consumed samples: 6656 | elapsed time per iteration (ms): 223304.1 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.13% | FLOPs estimator: workload-aware | logical tokens/s: 75131.6 | valid tokens/s: 75131.6 | physical tokens/s: 75825.3 | loss tokens/s: 56487.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.566142E-01 | lm-global-supervised-tokens: 1.261396E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.699154E-02 | lm-token-buffer-mib: 2.343209E+02 | global_load_balancing_loss: 1.000773E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11222.953 | params rms: 0.060769 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:58:29.653198 | finish at 2026-10-17 17:49:29 + [2026-10-12 02:54:41] iteration 209/ 2384 | consumed samples: 6688 | elapsed time per iteration (ms): 222058.2 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.29% | FLOPs estimator: workload-aware | logical tokens/s: 75553.2 | valid tokens/s: 75553.2 | physical tokens/s: 76324.5 | loss tokens/s: 56148.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.522104E-01 | lm-global-supervised-tokens: 1.246830E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.805438E-02 | lm-token-buffer-mib: 2.329965E+02 | global_load_balancing_loss: 1.001541E+00 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11222.886 | params rms: 0.060768 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:09:36.545852 | finish at 2026-10-17 17:04:18 + [2026-10-12 02:58:42] iteration 210/ 2384 | consumed samples: 6720 | elapsed time per iteration (ms): 240958.0 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.99% | FLOPs estimator: workload-aware | logical tokens/s: 69627.0 | valid tokens/s: 69627.0 | physical tokens/s: 70017.6 | loss tokens/s: 55477.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.728306E-01 | lm-global-supervised-tokens: 1.336770E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.477390E-02 | lm-token-buffer-mib: 2.325081E+02 | global_load_balancing_loss: 1.003092E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11222.819 | params rms: 0.060768 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:30:42.645749 | finish at 2026-10-18 04:29:25 + [2026-10-12 03:02:22] iteration 211/ 2384 | consumed samples: 6752 | elapsed time per iteration (ms): 219948.4 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.43% | FLOPs estimator: workload-aware | logical tokens/s: 76277.9 | valid tokens/s: 76277.9 | physical tokens/s: 76926.3 | loss tokens/s: 58164.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.841992E-01 | lm-global-supervised-tokens: 1.279308E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.029427E-02 | lm-token-buffer-mib: 2.330927E+02 | global_load_balancing_loss: 9.988627E-01 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11222.752 | params rms: 0.060768 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:45:47.883606 | finish at 2026-10-17 15:48:10 + [2026-10-12 03:06:06] iteration 212/ 2384 | consumed samples: 6784 | elapsed time per iteration (ms): 223659.5 | throughput per GPU (TFLOP/s/GPU): 19.4 | MFU 6.23% | FLOPs estimator: workload-aware | logical tokens/s: 75012.3 | valid tokens/s: 75012.3 | physical tokens/s: 75777.2 | loss tokens/s: 56459.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.991840E-01 | lm-global-supervised-tokens: 1.262773E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.681367E-02 | lm-token-buffer-mib: 2.327670E+02 | global_load_balancing_loss: 9.997126E-01 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11222.685 | params rms: 0.060767 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:56:28.377302 | finish at 2026-10-17 18:02:34 + [2026-10-12 03:09:52] iteration 213/ 2384 | consumed samples: 6816 | elapsed time per iteration (ms): 225629.0 | throughput per GPU (TFLOP/s/GPU): 24.9 | MFU 7.98% | FLOPs estimator: workload-aware | logical tokens/s: 74357.4 | valid tokens/s: 74357.4 | physical tokens/s: 74647.1 | loss tokens/s: 52786.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.392075E-01 | lm-global-supervised-tokens: 1.191019E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.435032E-02 | lm-token-buffer-mib: 2.326789E+02 | global_load_balancing_loss: 9.996629E-01 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11222.618 | params rms: 0.060767 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:04:00.577522 | finish at 2026-10-17 19:13:52 + [2026-10-12 03:13:34] iteration 214/ 2384 | consumed samples: 6848 | elapsed time per iteration (ms): 222726.0 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.31% | FLOPs estimator: workload-aware | logical tokens/s: 75326.6 | valid tokens/s: 75326.6 | physical tokens/s: 75759.8 | loss tokens/s: 56662.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.749768E-01 | lm-global-supervised-tokens: 1.262012E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.351684E-02 | lm-token-buffer-mib: 2.329218E+02 | global_load_balancing_loss: 1.000341E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11222.553 | params rms: 0.060766 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:15:15.314093 | finish at 2026-10-17 17:28:50 + [2026-10-12 03:17:19] iteration 215/ 2384 | consumed samples: 6880 | elapsed time per iteration (ms): 224948.6 | throughput per GPU (TFLOP/s/GPU): 24.4 | MFU 7.82% | FLOPs estimator: workload-aware | logical tokens/s: 74582.4 | valid tokens/s: 74582.4 | physical tokens/s: 75701.4 | loss tokens/s: 58547.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.798841E-01 | lm-global-supervised-tokens: 1.317017E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.967366E-02 | lm-token-buffer-mib: 2.337851E+02 | global_load_balancing_loss: 1.000380E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11222.487 | params rms: 0.060766 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:31:53.591603 | finish at 2026-10-17 18:49:13 + [2026-10-12 03:22:13] iteration 216/ 2384 | consumed samples: 6912 | elapsed time per iteration (ms): 293629.2 | throughput per GPU (TFLOP/s/GPU): 16.8 | MFU 5.38% | FLOPs estimator: workload-aware | logical tokens/s: 57137.4 | valid tokens/s: 57137.4 | physical tokens/s: 58549.2 | loss tokens/s: 44449.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.740965E-01 | lm-global-supervised-tokens: 1.305161E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.594679E-02 | lm-token-buffer-mib: 2.398241E+02 | global_load_balancing_loss: 9.998822E-01 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11222.420 | params rms: 0.060766 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 8:49:48.188263 | finish at 2026-10-19 12:12:01 + [2026-10-12 03:25:54] iteration 217/ 2384 | consumed samples: 6944 | elapsed time per iteration (ms): 221542.1 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.05% | FLOPs estimator: workload-aware | logical tokens/s: 75729.1 | valid tokens/s: 75729.1 | physical tokens/s: 76083.1 | loss tokens/s: 52968.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.026237E-01 | lm-global-supervised-tokens: 1.173476E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.161502E-02 | lm-token-buffer-mib: 2.334203E+02 | global_load_balancing_loss: 1.001711E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11222.350 | params rms: 0.060765 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:21:21.764697 | finish at 2026-10-17 16:47:16 + [2026-10-12 03:29:40] iteration 218/ 2384 | consumed samples: 6976 | elapsed time per iteration (ms): 225372.1 | throughput per GPU (TFLOP/s/GPU): 19.6 | MFU 6.29% | FLOPs estimator: workload-aware | logical tokens/s: 74442.2 | valid tokens/s: 74442.2 | physical tokens/s: 75551.8 | loss tokens/s: 56984.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.186422E-01 | lm-global-supervised-tokens: 1.284260E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.458181E-02 | lm-token-buffer-mib: 2.378692E+02 | global_load_balancing_loss: 1.000917E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11222.286 | params rms: 0.060765 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:35:55.974529 | finish at 2026-10-17 19:05:36 + [2026-10-12 03:33:22] iteration 219/ 2384 | consumed samples: 7008 | elapsed time per iteration (ms): 222474.3 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.79% | FLOPs estimator: workload-aware | logical tokens/s: 75411.9 | valid tokens/s: 75411.9 | physical tokens/s: 76427.1 | loss tokens/s: 56793.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.311388E-01 | lm-global-supervised-tokens: 1.263510E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.424050E-02 | lm-token-buffer-mib: 2.341876E+02 | global_load_balancing_loss: 1.000761E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11222.224 | params rms: 0.060765 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:47:36.888206 | finish at 2026-10-17 17:20:59 + [2026-10-12 03:37:25] iteration 220/ 2384 | consumed samples: 7040 | elapsed time per iteration (ms): 242790.7 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.92% | FLOPs estimator: workload-aware | logical tokens/s: 69101.5 | valid tokens/s: 69101.5 | physical tokens/s: 70042.3 | loss tokens/s: 54274.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.816531E-01 | lm-global-supervised-tokens: 1.317744E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.915410E-02 | lm-token-buffer-mib: 2.329748E+02 | global_load_balancing_loss: 1.002398E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11222.160 | params rms: 0.060764 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:56:39.155197 | finish at 2026-10-18 05:34:04 + [2026-10-12 03:41:08] iteration 221/ 2384 | consumed samples: 7072 | elapsed time per iteration (ms): 222735.8 | throughput per GPU (TFLOP/s/GPU): 19.3 | MFU 6.18% | FLOPs estimator: workload-aware | logical tokens/s: 75323.3 | valid tokens/s: 75323.3 | physical tokens/s: 76031.4 | loss tokens/s: 60734.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.628544E-01 | lm-global-supervised-tokens: 1.352783E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.748620E-02 | lm-token-buffer-mib: 2.337427E+02 | global_load_balancing_loss: 1.001939E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11222.100 | params rms: 0.060764 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:49:37.578263 | finish at 2026-10-17 17:30:45 + [2026-10-12 03:44:52] iteration 222/ 2384 | consumed samples: 7104 | elapsed time per iteration (ms): 223779.1 | throughput per GPU (TFLOP/s/GPU): 23.5 | MFU 7.54% | FLOPs estimator: workload-aware | logical tokens/s: 74972.1 | valid tokens/s: 74972.1 | physical tokens/s: 75849.9 | loss tokens/s: 55448.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.092676E-01 | lm-global-supervised-tokens: 1.240829E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.621618E-02 | lm-token-buffer-mib: 2.331310E+02 | global_load_balancing_loss: 1.000485E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11222.037 | params rms: 0.060764 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:23:30.512011 | finish at 2026-10-17 18:08:22 + [2026-10-12 03:48:36] iteration 223/ 2384 | consumed samples: 7136 | elapsed time per iteration (ms): 224480.5 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.85% | FLOPs estimator: workload-aware | logical tokens/s: 74737.7 | valid tokens/s: 74737.7 | physical tokens/s: 75600.4 | loss tokens/s: 58983.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.806692E-01 | lm-global-supervised-tokens: 1.324058E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.678045E-02 | lm-token-buffer-mib: 2.350955E+02 | global_load_balancing_loss: 9.999196E-01 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11221.971 | params rms: 0.060763 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:45:02.467629 | finish at 2026-10-17 18:33:39 + [2026-10-12 03:52:19] iteration 224/ 2384 | consumed samples: 7168 | elapsed time per iteration (ms): 222756.9 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.89% | FLOPs estimator: workload-aware | logical tokens/s: 75316.2 | valid tokens/s: 75316.2 | physical tokens/s: 75975.6 | loss tokens/s: 58289.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.800745E-01 | lm-global-supervised-tokens: 1.298435E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.819039E-02 | lm-token-buffer-mib: 2.345651E+02 | global_load_balancing_loss: 9.997686E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11221.908 | params rms: 0.060763 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:39:14.916000 | finish at 2026-10-17 17:31:34 + [2026-10-12 03:56:02] iteration 225/ 2384 | consumed samples: 7200 | elapsed time per iteration (ms): 222774.0 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.76% | FLOPs estimator: workload-aware | logical tokens/s: 75310.4 | valid tokens/s: 75310.4 | physical tokens/s: 75713.8 | loss tokens/s: 54884.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.360832E-01 | lm-global-supervised-tokens: 1.222672E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.739544E-02 | lm-token-buffer-mib: 2.340612E+02 | global_load_balancing_loss: 9.996454E-01 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11221.846 | params rms: 0.060763 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:36:09.031360 | finish at 2026-10-17 17:32:11 + [2026-10-12 03:59:46] iteration 226/ 2384 | consumed samples: 7232 | elapsed time per iteration (ms): 224022.4 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.06% | FLOPs estimator: workload-aware | logical tokens/s: 74890.7 | valid tokens/s: 74890.7 | physical tokens/s: 75782.4 | loss tokens/s: 54508.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.159640E-01 | lm-global-supervised-tokens: 1.221116E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.790946E-02 | lm-token-buffer-mib: 2.355098E+02 | global_load_balancing_loss: 1.000809E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11221.786 | params rms: 0.060762 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:17:20.355968 | finish at 2026-10-17 18:17:06 + [2026-10-12 04:03:28] iteration 227/ 2384 | consumed samples: 7264 | elapsed time per iteration (ms): 222070.4 | throughput per GPU (TFLOP/s/GPU): 18.6 | MFU 5.96% | FLOPs estimator: workload-aware | logical tokens/s: 75549.0 | valid tokens/s: 75549.0 | physical tokens/s: 76667.0 | loss tokens/s: 57168.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.143984E-01 | lm-global-supervised-tokens: 1.269537E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.786820E-02 | lm-token-buffer-mib: 2.339737E+02 | global_load_balancing_loss: 1.001479E+00 | loss scale: 1.0 | grad norm: 0.032 | params norm: 11221.721 | params rms: 0.060762 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:03:25.854856 | finish at 2026-10-17 17:06:54 + [2026-10-12 04:08:18] iteration 228/ 2384 | consumed samples: 7296 | elapsed time per iteration (ms): 289874.8 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.42% | FLOPs estimator: workload-aware | logical tokens/s: 57877.4 | valid tokens/s: 57877.4 | physical tokens/s: 59338.0 | loss tokens/s: 42028.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.192205E-01 | lm-global-supervised-tokens: 1.218289E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.270769E-02 | lm-token-buffer-mib: 2.428734E+02 | global_load_balancing_loss: 1.001719E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11221.659 | params rms: 0.060762 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 5:36:10.018867 | finish at 2026-10-19 09:44:28 + [2026-10-12 04:11:58] iteration 229/ 2384 | consumed samples: 7328 | elapsed time per iteration (ms): 220678.1 | throughput per GPU (TFLOP/s/GPU): 25.7 | MFU 8.25% | FLOPs estimator: workload-aware | logical tokens/s: 76025.7 | valid tokens/s: 76025.7 | physical tokens/s: 76436.1 | loss tokens/s: 61764.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.274638E-01 | lm-global-supervised-tokens: 1.363002E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 5.122434E-02 | lm-token-buffer-mib: 2.335746E+02 | global_load_balancing_loss: 1.002284E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11221.598 | params rms: 0.060761 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:06:01.404897 | finish at 2026-10-17 16:18:00 + [2026-10-12 04:16:03] iteration 230/ 2384 | consumed samples: 7360 | elapsed time per iteration (ms): 244757.9 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.58% | FLOPs estimator: workload-aware | logical tokens/s: 68545.9 | valid tokens/s: 68545.9 | physical tokens/s: 69327.2 | loss tokens/s: 50325.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.541623E-01 | lm-global-supervised-tokens: 1.231767E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.445553E-02 | lm-token-buffer-mib: 2.351856E+02 | global_load_balancing_loss: 9.998491E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11221.540 | params rms: 0.060761 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 2:26:48.561278 | finish at 2026-10-18 06:42:52 + [2026-10-12 04:19:43] iteration 231/ 2384 | consumed samples: 7392 | elapsed time per iteration (ms): 220387.9 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.54% | FLOPs estimator: workload-aware | logical tokens/s: 76125.7 | valid tokens/s: 76125.7 | physical tokens/s: 76847.5 | loss tokens/s: 55515.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.321357E-01 | lm-global-supervised-tokens: 1.223495E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.030552E-02 | lm-token-buffer-mib: 2.356844E+02 | global_load_balancing_loss: 1.000291E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11221.480 | params rms: 0.060761 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:48:15.151512 | finish at 2026-10-17 16:07:59 + [2026-10-12 04:23:26] iteration 232/ 2384 | consumed samples: 7424 | elapsed time per iteration (ms): 222075.9 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.58% | FLOPs estimator: workload-aware | logical tokens/s: 75547.2 | valid tokens/s: 75547.2 | physical tokens/s: 75974.0 | loss tokens/s: 57196.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.148532E-01 | lm-global-supervised-tokens: 1.270199E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.238968E-02 | lm-token-buffer-mib: 2.334944E+02 | global_load_balancing_loss: 1.001493E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11221.421 | params rms: 0.060760 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:45:07.313366 | finish at 2026-10-17 17:08:33 + [2026-10-12 04:27:08] iteration 233/ 2384 | consumed samples: 7456 | elapsed time per iteration (ms): 222018.5 | throughput per GPU (TFLOP/s/GPU): 25.8 | MFU 8.26% | FLOPs estimator: workload-aware | logical tokens/s: 75566.6 | valid tokens/s: 75566.6 | physical tokens/s: 75863.1 | loss tokens/s: 53875.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.530282E-01 | lm-global-supervised-tokens: 1.196142E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.544427E-02 | lm-token-buffer-mib: 2.323965E+02 | global_load_balancing_loss: 1.000356E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11221.368 | params rms: 0.060760 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:39:21.797283 | finish at 2026-10-17 17:06:29 + [2026-10-12 04:30:50] iteration 234/ 2384 | consumed samples: 7488 | elapsed time per iteration (ms): 222148.9 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.37% | FLOPs estimator: workload-aware | logical tokens/s: 75522.3 | valid tokens/s: 75522.3 | physical tokens/s: 76060.5 | loss tokens/s: 53150.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.505441E-01 | lm-global-supervised-tokens: 1.180729E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.862340E-02 | lm-token-buffer-mib: 2.344127E+02 | global_load_balancing_loss: 9.996229E-01 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11221.321 | params rms: 0.060760 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:40:20.185816 | finish at 2026-10-17 17:11:10 + [2026-10-12 04:34:31] iteration 235/ 2384 | consumed samples: 7520 | elapsed time per iteration (ms): 221158.8 | throughput per GPU (TFLOP/s/GPU): 25.7 | MFU 8.22% | FLOPs estimator: workload-aware | logical tokens/s: 75860.3 | valid tokens/s: 75860.3 | physical tokens/s: 76576.4 | loss tokens/s: 55717.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.217141E-01 | lm-global-supervised-tokens: 1.232249E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.240907E-02 | lm-token-buffer-mib: 2.323340E+02 | global_load_balancing_loss: 9.984648E-01 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11221.275 | params rms: 0.060760 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:01:10.350620 | finish at 2026-10-17 16:35:41 + [2026-10-12 04:38:12] iteration 236/ 2384 | consumed samples: 7552 | elapsed time per iteration (ms): 221307.1 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.24% | FLOPs estimator: workload-aware | logical tokens/s: 75809.6 | valid tokens/s: 75809.6 | physical tokens/s: 76446.7 | loss tokens/s: 60257.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.592721E-01 | lm-global-supervised-tokens: 1.333545E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.475989E-02 | lm-token-buffer-mib: 2.349065E+02 | global_load_balancing_loss: 1.001926E+00 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11221.223 | params rms: 0.060759 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:02:47.698039 | finish at 2026-10-17 16:41:00 + [2026-10-12 04:41:54] iteration 237/ 2384 | consumed samples: 7584 | elapsed time per iteration (ms): 221814.8 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.17% | FLOPs estimator: workload-aware | logical tokens/s: 75635.9 | valid tokens/s: 75635.9 | physical tokens/s: 76424.4 | loss tokens/s: 57902.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.440415E-01 | lm-global-supervised-tokens: 1.284358E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.774284E-02 | lm-token-buffer-mib: 2.353101E+02 | global_load_balancing_loss: 1.001332E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11221.172 | params rms: 0.060759 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:17:16.429399 | finish at 2026-10-17 16:59:10 + [2026-10-12 04:45:34] iteration 238/ 2384 | consumed samples: 7616 | elapsed time per iteration (ms): 220279.0 | throughput per GPU (TFLOP/s/GPU): 20.2 | MFU 6.47% | FLOPs estimator: workload-aware | logical tokens/s: 76163.3 | valid tokens/s: 76163.3 | physical tokens/s: 76522.0 | loss tokens/s: 57121.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.683069E-01 | lm-global-supervised-tokens: 1.258271E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.459551E-02 | lm-token-buffer-mib: 2.336219E+02 | global_load_balancing_loss: 1.001555E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11221.123 | params rms: 0.060759 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:18:38.744839 | finish at 2026-10-17 16:04:13 + [2026-10-12 04:49:16] iteration 239/ 2384 | consumed samples: 7648 | elapsed time per iteration (ms): 221831.0 | throughput per GPU (TFLOP/s/GPU): 23.8 | MFU 7.64% | FLOPs estimator: workload-aware | logical tokens/s: 75630.4 | valid tokens/s: 75630.4 | physical tokens/s: 76174.4 | loss tokens/s: 59178.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.969444E-01 | lm-global-supervised-tokens: 1.312754E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.443918E-02 | lm-token-buffer-mib: 2.348155E+02 | global_load_balancing_loss: 1.004305E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11221.077 | params rms: 0.060758 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:10:27.566789 | finish at 2026-10-17 16:59:44 + [2026-10-12 04:53:20] iteration 240/ 2384 | consumed samples: 7680 | elapsed time per iteration (ms): 244358.8 | throughput per GPU (TFLOP/s/GPU): 19.1 | MFU 6.11% | FLOPs estimator: workload-aware | logical tokens/s: 68658.1 | valid tokens/s: 68658.1 | physical tokens/s: 69317.6 | loss tokens/s: 52898.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.785318E-01 | lm-global-supervised-tokens: 1.292631E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.923766E-02 | lm-token-buffer-mib: 2.338378E+02 | global_load_balancing_loss: 1.000487E+00 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11221.031 | params rms: 0.060758 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:31:45.180672 | finish at 2026-10-18 06:25:06 +times across ranks (ms): + save-checkpoint: + rank 0: 24139.97 + rank 1: 24139.96 + rank 2: 24139.95 + rank 3: 24139.99 + rank 4: 24139.97 + rank 5: 24139.96 + rank 6: 24139.96 + rank 7: 24139.98 + rank 8: 24139.62 + rank 9: 24139.63 + rank 10: 24139.62 + rank 11: 24139.62 + rank 12: 24139.61 + rank 13: 24139.63 + rank 14: 24139.64 + rank 15: 24139.64 + rank 16: 24139.68 + rank 17: 24139.69 + rank 18: 24139.69 + rank 19: 24139.69 + rank 20: 24139.68 + rank 21: 24139.69 + rank 22: 24139.65 + rank 23: 24139.67 + rank 24: 24139.32 + rank 25: 24139.31 + rank 26: 24139.31 + rank 27: 24139.34 + rank 28: 24139.31 + rank 29: 24139.35 + rank 30: 24139.39 + rank 31: 24139.32 + rank 32: 24139.36 + rank 33: 24139.32 + rank 34: 24139.36 + rank 35: 24139.41 + rank 36: 24139.32 + rank 37: 24139.36 + rank 38: 24139.31 + rank 39: 24139.30 + rank 40: 24139.51 + rank 41: 24139.45 + rank 42: 24139.43 + rank 43: 24139.44 + rank 44: 24139.43 + rank 45: 24139.43 + rank 46: 24139.47 + rank 47: 24139.44 + rank 48: 24139.46 + rank 49: 24139.38 + rank 50: 24139.40 + rank 51: 24139.37 + rank 52: 24139.36 + rank 53: 24139.41 + rank 54: 24139.35 + rank 55: 24139.35 + rank 56: 24139.65 + rank 57: 24139.66 + rank 58: 24139.65 + rank 59: 24139.63 + rank 60: 24139.63 + rank 61: 24139.63 + rank 62: 24139.62 + rank 63: 24139.62 + [2026-10-12 04:57:33] iteration 241/ 2384 | consumed samples: 7712 | elapsed time per iteration (ms): 228646.2 | throughput per GPU (TFLOP/s/GPU): 24.2 | MFU 7.76% | FLOPs estimator: workload-aware | logical tokens/s: 73376.2 | valid tokens/s: 73376.2 | physical tokens/s: 74442.5 | loss tokens/s: 57246.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.000616E-01 | lm-global-supervised-tokens: 1.308920E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 5.116298E-01 | lm-token-buffer-mib: 2.343742E+02 | global_load_balancing_loss: 9.996887E-01 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11220.982 | params rms: 0.060758 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:06:28.863188 | finish at 2026-10-17 21:04:02 + [2026-10-12 05:01:17] iteration 242/ 2384 | consumed samples: 7744 | elapsed time per iteration (ms): 224119.5 | throughput per GPU (TFLOP/s/GPU): 19.3 | MFU 6.20% | FLOPs estimator: workload-aware | logical tokens/s: 74858.2 | valid tokens/s: 74858.2 | physical tokens/s: 75588.0 | loss tokens/s: 54415.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.162785E-01 | lm-global-supervised-tokens: 1.219564E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.340370E-02 | lm-token-buffer-mib: 2.331894E+02 | global_load_balancing_loss: 9.998192E-01 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11220.935 | params rms: 0.060758 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:21:03.985175 | finish at 2026-10-17 18:22:21 + [2026-10-12 05:04:58] iteration 243/ 2384 | consumed samples: 7776 | elapsed time per iteration (ms): 220242.6 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.78% | FLOPs estimator: workload-aware | logical tokens/s: 76176.0 | valid tokens/s: 76176.0 | physical tokens/s: 76606.4 | loss tokens/s: 57870.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.555362E-01 | lm-global-supervised-tokens: 1.274557E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.583848E-02 | lm-token-buffer-mib: 2.337889E+02 | global_load_balancing_loss: 9.994828E-01 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11220.888 | params rms: 0.060757 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 10:58:59.342206 | finish at 2026-10-17 16:03:57 + [2026-10-12 05:08:39] iteration 244/ 2384 | consumed samples: 7808 | elapsed time per iteration (ms): 221546.8 | throughput per GPU (TFLOP/s/GPU): 23.7 | MFU 7.59% | FLOPs estimator: workload-aware | logical tokens/s: 75727.5 | valid tokens/s: 75727.5 | physical tokens/s: 77056.2 | loss tokens/s: 58477.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.007762E-01 | lm-global-supervised-tokens: 1.295554E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.615956E-02 | lm-token-buffer-mib: 2.348928E+02 | global_load_balancing_loss: 9.991294E-01 | loss scale: 1.0 | grad norm: 0.167 | params norm: 11220.842 | params rms: 0.060757 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:41:50.215049 | finish at 2026-10-17 16:50:29 + [2026-10-12 05:12:22] iteration 245/ 2384 | consumed samples: 7840 | elapsed time per iteration (ms): 222339.0 | throughput per GPU (TFLOP/s/GPU): 25.4 | MFU 8.15% | FLOPs estimator: workload-aware | logical tokens/s: 75457.7 | valid tokens/s: 75457.7 | physical tokens/s: 76147.2 | loss tokens/s: 56860.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.302092E-01 | lm-global-supervised-tokens: 1.264219E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.602476E-02 | lm-token-buffer-mib: 2.335096E+02 | global_load_balancing_loss: 9.998875E-01 | loss scale: 1.0 | grad norm: 0.186 | params norm: 11220.792 | params rms: 0.060757 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:06:23.080173 | finish at 2026-10-17 17:18:45 + [2026-10-12 05:16:01] iteration 246/ 2384 | consumed samples: 7872 | elapsed time per iteration (ms): 219974.8 | throughput per GPU (TFLOP/s/GPU): 18.9 | MFU 6.07% | FLOPs estimator: workload-aware | logical tokens/s: 76268.7 | valid tokens/s: 76268.7 | physical tokens/s: 77475.7 | loss tokens/s: 60623.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.789813E-01 | lm-global-supervised-tokens: 1.333554E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.536498E-02 | lm-token-buffer-mib: 2.346447E+02 | global_load_balancing_loss: 9.996771E-01 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11220.739 | params rms: 0.060757 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 10:38:26.037508 | finish at 2026-10-17 15:54:28 + [2026-10-12 05:19:43] iteration 247/ 2384 | consumed samples: 7904 | elapsed time per iteration (ms): 221542.9 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.15% | FLOPs estimator: workload-aware | logical tokens/s: 75728.8 | valid tokens/s: 75728.8 | physical tokens/s: 76660.3 | loss tokens/s: 60036.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.811696E-01 | lm-global-supervised-tokens: 1.330064E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.285958E-02 | lm-token-buffer-mib: 2.346760E+02 | global_load_balancing_loss: 1.001471E+00 | loss scale: 1.0 | grad norm: 0.158 | params norm: 11220.690 | params rms: 0.060756 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:30:37.270212 | finish at 2026-10-17 16:50:20 + [2026-10-12 05:23:25] iteration 248/ 2384 | consumed samples: 7936 | elapsed time per iteration (ms): 222191.4 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.03% | FLOPs estimator: workload-aware | logical tokens/s: 75507.9 | valid tokens/s: 75507.9 | physical tokens/s: 75918.4 | loss tokens/s: 58361.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.011032E-01 | lm-global-supervised-tokens: 1.296731E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.983270E-02 | lm-token-buffer-mib: 2.324781E+02 | global_load_balancing_loss: 1.000843E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11220.638 | params rms: 0.060756 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:50:00.792315 | finish at 2026-10-17 17:13:26 + [2026-10-12 05:27:09] iteration 249/ 2384 | consumed samples: 7968 | elapsed time per iteration (ms): 223478.1 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.83% | FLOPs estimator: workload-aware | logical tokens/s: 75073.1 | valid tokens/s: 75073.1 | physical tokens/s: 76116.2 | loss tokens/s: 58240.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.139420E-01 | lm-global-supervised-tokens: 1.301535E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.057443E-02 | lm-token-buffer-mib: 2.356631E+02 | global_load_balancing_loss: 1.001475E+00 | loss scale: 1.0 | grad norm: 0.123 | params norm: 11220.587 | params rms: 0.060756 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:32:05.654043 | finish at 2026-10-17 17:59:14 + [2026-10-12 05:31:16] iteration 250/ 2384 | consumed samples: 8000 | elapsed time per iteration (ms): 247706.8 | throughput per GPU (TFLOP/s/GPU): 18.7 | MFU 6.01% | FLOPs estimator: workload-aware | logical tokens/s: 67730.0 | valid tokens/s: 67730.0 | physical tokens/s: 68866.7 | loss tokens/s: 53434.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.286543E-01 | lm-global-supervised-tokens: 1.323601E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.228350E-02 | lm-token-buffer-mib: 2.377425E+02 | global_load_balancing_loss: 1.001037E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11220.539 | params rms: 0.060756 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 2:50:06.296411 | finish at 2026-10-18 08:21:23 + [2026-10-12 05:35:00] iteration 251/ 2384 | consumed samples: 8032 | elapsed time per iteration (ms): 223560.2 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.65% | FLOPs estimator: workload-aware | logical tokens/s: 75045.5 | valid tokens/s: 75045.5 | physical tokens/s: 75921.4 | loss tokens/s: 60466.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.516382E-01 | lm-global-supervised-tokens: 1.351793E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.738134E-02 | lm-token-buffer-mib: 2.345603E+02 | global_load_balancing_loss: 1.000954E+00 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11220.492 | params rms: 0.060755 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:27:33.847557 | finish at 2026-10-17 18:02:34 + [2026-10-12 05:38:45] iteration 252/ 2384 | consumed samples: 8064 | elapsed time per iteration (ms): 225536.5 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.57% | FLOPs estimator: workload-aware | logical tokens/s: 74387.8 | valid tokens/s: 74387.8 | physical tokens/s: 75418.7 | loss tokens/s: 53401.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.995020E-01 | lm-global-supervised-tokens: 1.204400E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.145633E-02 | lm-token-buffer-mib: 2.352197E+02 | global_load_balancing_loss: 1.000214E+00 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11220.441 | params rms: 0.060755 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:34:03.859632 | finish at 2026-10-17 19:12:49 + [2026-10-12 05:42:27] iteration 253/ 2384 | consumed samples: 8096 | elapsed time per iteration (ms): 221296.3 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.11% | FLOPs estimator: workload-aware | logical tokens/s: 75813.1 | valid tokens/s: 75813.1 | physical tokens/s: 76344.9 | loss tokens/s: 58226.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.369431E-01 | lm-global-supervised-tokens: 1.288520E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.978287E-02 | lm-token-buffer-mib: 2.334887E+02 | global_load_balancing_loss: 1.000939E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11220.391 | params rms: 0.060755 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 10:59:42.451741 | finish at 2026-10-17 16:42:09 + [2026-10-12 05:46:15] iteration 254/ 2384 | consumed samples: 8128 | elapsed time per iteration (ms): 228400.2 | throughput per GPU (TFLOP/s/GPU): 25.5 | MFU 8.18% | FLOPs estimator: workload-aware | logical tokens/s: 73455.1 | valid tokens/s: 73455.1 | physical tokens/s: 74387.6 | loss tokens/s: 55453.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.441933E-01 | lm-global-supervised-tokens: 1.266559E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.265207E-02 | lm-token-buffer-mib: 2.351319E+02 | global_load_balancing_loss: 9.999667E-01 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11220.341 | params rms: 0.060754 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:08:12.497878 | finish at 2026-10-17 20:54:28 + [2026-10-12 05:49:58] iteration 255/ 2384 | consumed samples: 8160 | elapsed time per iteration (ms): 222335.5 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.65% | FLOPs estimator: workload-aware | logical tokens/s: 75458.8 | valid tokens/s: 75458.8 | physical tokens/s: 76614.7 | loss tokens/s: 57783.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.789344E-01 | lm-global-supervised-tokens: 1.284726E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.521435E-02 | lm-token-buffer-mib: 2.360622E+02 | global_load_balancing_loss: 1.000430E+00 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11220.282 | params rms: 0.060754 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:29:12.345999 | finish at 2026-10-17 17:19:10 + [2026-10-12 05:53:40] iteration 256/ 2384 | consumed samples: 8192 | elapsed time per iteration (ms): 222795.5 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.57% | FLOPs estimator: workload-aware | logical tokens/s: 75303.2 | valid tokens/s: 75303.2 | physical tokens/s: 75663.1 | loss tokens/s: 57312.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.671052E-01 | lm-global-supervised-tokens: 1.276892E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.774108E-02 | lm-token-buffer-mib: 2.329684E+02 | global_load_balancing_loss: 9.994578E-01 | loss scale: 1.0 | grad norm: 0.095 | params norm: 11220.224 | params rms: 0.060754 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:41:48.798210 | finish at 2026-10-17 17:35:29 + [2026-10-12 05:57:24] iteration 257/ 2384 | consumed samples: 8224 | elapsed time per iteration (ms): 223290.9 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.79% | FLOPs estimator: workload-aware | logical tokens/s: 75136.1 | valid tokens/s: 75136.1 | physical tokens/s: 75889.1 | loss tokens/s: 55383.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.836191E-01 | lm-global-supervised-tokens: 1.236653E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.950451E-02 | lm-token-buffer-mib: 2.346905E+02 | global_load_balancing_loss: 9.997501E-01 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11220.168 | params rms: 0.060754 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:55:39.738705 | finish at 2026-10-17 17:53:03 + [2026-10-12 06:01:05] iteration 258/ 2384 | consumed samples: 8256 | elapsed time per iteration (ms): 221165.3 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.07% | FLOPs estimator: workload-aware | logical tokens/s: 75858.2 | valid tokens/s: 75858.2 | physical tokens/s: 76582.6 | loss tokens/s: 57721.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.430092E-01 | lm-global-supervised-tokens: 1.276597E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.614901E-02 | lm-token-buffer-mib: 2.353087E+02 | global_load_balancing_loss: 1.000750E+00 | loss scale: 1.0 | grad norm: 0.154 | params norm: 11220.113 | params rms: 0.060753 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 10:36:37.354074 | finish at 2026-10-17 16:37:42 + [2026-10-12 06:04:48] iteration 259/ 2384 | consumed samples: 8288 | elapsed time per iteration (ms): 223161.4 | throughput per GPU (TFLOP/s/GPU): 25.9 | MFU 8.29% | FLOPs estimator: workload-aware | logical tokens/s: 75179.7 | valid tokens/s: 75179.6 | physical tokens/s: 75653.2 | loss tokens/s: 53428.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.592942E-01 | lm-global-supervised-tokens: 1.192321E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.683256E-02 | lm-token-buffer-mib: 2.344889E+02 | global_load_balancing_loss: 1.002049E+00 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11220.057 | params rms: 0.060753 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:43:37.888534 | finish at 2026-10-17 17:48:26 + [2026-10-12 06:08:48] iteration 260/ 2384 | consumed samples: 8320 | elapsed time per iteration (ms): 240451.6 | throughput per GPU (TFLOP/s/GPU): 19.0 | MFU 6.09% | FLOPs estimator: workload-aware | logical tokens/s: 69773.6 | valid tokens/s: 69773.6 | physical tokens/s: 70691.5 | loss tokens/s: 55138.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.519764E-01 | lm-global-supervised-tokens: 1.325810E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 5.112807E-02 | lm-token-buffer-mib: 2.338159E+02 | global_load_balancing_loss: 9.992888E-01 | loss scale: 1.0 | grad norm: 0.223 | params norm: 11219.999 | params rms: 0.060753 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:51:59.097279 | finish at 2026-10-18 04:00:47 + [2026-10-12 06:12:27] iteration 261/ 2384 | consumed samples: 8352 | elapsed time per iteration (ms): 218559.9 | throughput per GPU (TFLOP/s/GPU): 19.1 | MFU 6.14% | FLOPs estimator: workload-aware | logical tokens/s: 76762.6 | valid tokens/s: 76762.6 | physical tokens/s: 77216.4 | loss tokens/s: 57024.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.175370E-01 | lm-global-supervised-tokens: 1.246316E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.586534E-02 | lm-token-buffer-mib: 2.328784E+02 | global_load_balancing_loss: 1.000715E+00 | loss scale: 1.0 | grad norm: 0.148 | params norm: 11219.941 | params rms: 0.060752 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:53:22.640464 | finish at 2026-10-17 15:05:50 + [2026-10-12 06:16:11] iteration 262/ 2384 | consumed samples: 8384 | elapsed time per iteration (ms): 223573.8 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.97% | FLOPs estimator: workload-aware | logical tokens/s: 75041.0 | valid tokens/s: 75041.0 | physical tokens/s: 75765.6 | loss tokens/s: 53631.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.977711E-01 | lm-global-supervised-tokens: 1.199066E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.618423E-02 | lm-token-buffer-mib: 2.350622E+02 | global_load_balancing_loss: 1.001930E+00 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11219.883 | params rms: 0.060752 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:47:03.624022 | finish at 2026-10-17 18:03:14 + [2026-10-12 06:19:53] iteration 263/ 2384 | consumed samples: 8416 | elapsed time per iteration (ms): 222743.5 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.75% | FLOPs estimator: workload-aware | logical tokens/s: 75320.8 | valid tokens/s: 75320.8 | physical tokens/s: 76020.7 | loss tokens/s: 59067.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.802726E-01 | lm-global-supervised-tokens: 1.315687E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.283065E-02 | lm-token-buffer-mib: 2.330451E+02 | global_load_balancing_loss: 1.000290E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11219.827 | params rms: 0.060752 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:13:58.927078 | finish at 2026-10-17 17:33:52 + [2026-10-12 06:23:37] iteration 264/ 2384 | consumed samples: 8448 | elapsed time per iteration (ms): 223971.3 | throughput per GPU (TFLOP/s/GPU): 23.7 | MFU 7.59% | FLOPs estimator: workload-aware | logical tokens/s: 74907.8 | valid tokens/s: 74907.8 | physical tokens/s: 75796.6 | loss tokens/s: 58535.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.806686E-01 | lm-global-supervised-tokens: 1.311018E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.634159E-02 | lm-token-buffer-mib: 2.347899E+02 | global_load_balancing_loss: 1.000091E+00 | loss scale: 1.0 | grad norm: 0.189 | params norm: 11219.772 | params rms: 0.060751 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:53:39.161825 | finish at 2026-10-17 18:17:16 + [2026-10-12 06:27:20] iteration 265/ 2384 | consumed samples: 8480 | elapsed time per iteration (ms): 223091.7 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.73% | FLOPs estimator: workload-aware | logical tokens/s: 75203.1 | valid tokens/s: 75203.1 | physical tokens/s: 75821.2 | loss tokens/s: 56842.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.602536E-01 | lm-global-supervised-tokens: 1.268117E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.944335E-02 | lm-token-buffer-mib: 2.325260E+02 | global_load_balancing_loss: 1.000648E+00 | loss scale: 1.0 | grad norm: 0.162 | params norm: 11219.717 | params rms: 0.060751 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:18:51.277099 | finish at 2026-10-17 17:46:12 + [2026-10-12 06:31:03] iteration 266/ 2384 | consumed samples: 8512 | elapsed time per iteration (ms): 223140.2 | throughput per GPU (TFLOP/s/GPU): 19.9 | MFU 6.37% | FLOPs estimator: workload-aware | logical tokens/s: 75186.8 | valid tokens/s: 75186.8 | physical tokens/s: 75654.4 | loss tokens/s: 58490.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.475256E-01 | lm-global-supervised-tokens: 1.305168E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.777215E-02 | lm-token-buffer-mib: 2.336024E+02 | global_load_balancing_loss: 1.000295E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11219.667 | params rms: 0.060751 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:16:51.014083 | finish at 2026-10-17 17:47:54 + [2026-10-12 06:34:49] iteration 267/ 2384 | consumed samples: 8544 | elapsed time per iteration (ms): 225440.9 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.85% | FLOPs estimator: workload-aware | logical tokens/s: 74419.4 | valid tokens/s: 74419.4 | physical tokens/s: 75565.1 | loss tokens/s: 57100.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.871708E-01 | lm-global-supervised-tokens: 1.287277E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.473239E-02 | lm-token-buffer-mib: 2.380558E+02 | global_load_balancing_loss: 1.001686E+00 | loss scale: 1.0 | grad norm: 0.162 | params norm: 11219.617 | params rms: 0.060751 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:34:18.430911 | finish at 2026-10-17 19:09:07 + [2026-10-12 06:38:30] iteration 268/ 2384 | consumed samples: 8576 | elapsed time per iteration (ms): 221209.8 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.03% | FLOPs estimator: workload-aware | logical tokens/s: 75842.9 | valid tokens/s: 75842.9 | physical tokens/s: 76247.5 | loss tokens/s: 56420.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.733246E-01 | lm-global-supervised-tokens: 1.248081E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 5.305457E-02 | lm-token-buffer-mib: 2.335947E+02 | global_load_balancing_loss: 9.998124E-01 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11219.566 | params rms: 0.060750 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 10:01:19.899982 | finish at 2026-10-17 16:39:50 + [2026-10-12 06:42:11] iteration 269/ 2384 | consumed samples: 8608 | elapsed time per iteration (ms): 220440.0 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.98% | FLOPs estimator: workload-aware | logical tokens/s: 76107.8 | valid tokens/s: 76107.7 | physical tokens/s: 77084.8 | loss tokens/s: 55729.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.760448E-01 | lm-global-supervised-tokens: 1.228501E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.110591E-02 | lm-token-buffer-mib: 2.337684E+02 | global_load_balancing_loss: 9.997844E-01 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11219.515 | params rms: 0.060750 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 9:30:30.549695 | finish at 2026-10-17 16:12:41 + [2026-10-12 06:47:07] iteration 270/ 2384 | consumed samples: 8640 | elapsed time per iteration (ms): 296676.9 | throughput per GPU (TFLOP/s/GPU): 19.1 | MFU 6.13% | FLOPs estimator: workload-aware | logical tokens/s: 56550.4 | valid tokens/s: 56550.4 | physical tokens/s: 57608.4 | loss tokens/s: 38999.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.936493E-01 | lm-global-supervised-tokens: 1.157017E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.578068E-02 | lm-token-buffer-mib: 2.340768E+02 | global_load_balancing_loss: 1.000514E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11219.464 | params rms: 0.060750 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 6:12:54.962378 | finish at 2026-10-19 13:00:02 + [2026-10-12 06:50:51] iteration 271/ 2384 | consumed samples: 8672 | elapsed time per iteration (ms): 223886.2 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.63% | FLOPs estimator: workload-aware | logical tokens/s: 74936.3 | valid tokens/s: 74936.3 | physical tokens/s: 76105.0 | loss tokens/s: 58966.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.757211E-01 | lm-global-supervised-tokens: 1.320188E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.286948E-02 | lm-token-buffer-mib: 2.357536E+02 | global_load_balancing_loss: 1.002124E+00 | loss scale: 1.0 | grad norm: 0.194 | params norm: 11219.413 | params rms: 0.060749 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:24:31.537474 | finish at 2026-10-17 18:15:23 + [2026-10-12 06:54:37] iteration 272/ 2384 | consumed samples: 8704 | elapsed time per iteration (ms): 226130.3 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.39% | FLOPs estimator: workload-aware | logical tokens/s: 74192.4 | valid tokens/s: 74192.4 | physical tokens/s: 74813.9 | loss tokens/s: 55825.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.720490E-01 | lm-global-supervised-tokens: 1.262391E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.575923E-02 | lm-token-buffer-mib: 2.328498E+02 | global_load_balancing_loss: 1.002040E+00 | loss scale: 1.0 | grad norm: 0.115 | params norm: 11219.362 | params rms: 0.060749 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:39:47.190674 | finish at 2026-10-17 19:34:24 + [2026-10-12 06:58:16] iteration 273/ 2384 | consumed samples: 8736 | elapsed time per iteration (ms): 218933.3 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.31% | FLOPs estimator: workload-aware | logical tokens/s: 76631.5 | valid tokens/s: 76631.4 | physical tokens/s: 77017.2 | loss tokens/s: 54338.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.216733E-01 | lm-global-supervised-tokens: 1.189642E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.343298E-02 | lm-token-buffer-mib: 2.330509E+02 | global_load_balancing_loss: 1.002254E+00 | loss scale: 1.0 | grad norm: 0.112 | params norm: 11219.311 | params rms: 0.060749 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:22:48.193319 | finish at 2026-10-17 15:21:04 + [2026-10-12 07:01:59] iteration 274/ 2384 | consumed samples: 8768 | elapsed time per iteration (ms): 223166.0 | throughput per GPU (TFLOP/s/GPU): 18.9 | MFU 6.05% | FLOPs estimator: workload-aware | logical tokens/s: 75178.1 | valid tokens/s: 75178.1 | physical tokens/s: 76687.5 | loss tokens/s: 56379.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.890970E-01 | lm-global-supervised-tokens: 1.258190E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.315738E-02 | lm-token-buffer-mib: 2.387961E+02 | global_load_balancing_loss: 1.001203E+00 | loss scale: 1.0 | grad norm: 0.124 | params norm: 11219.262 | params rms: 0.060749 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 10:48:00.219522 | finish at 2026-10-17 17:50:00 + [2026-10-12 07:05:39] iteration 275/ 2384 | consumed samples: 8800 | elapsed time per iteration (ms): 219815.8 | throughput per GPU (TFLOP/s/GPU): 25.9 | MFU 8.29% | FLOPs estimator: workload-aware | logical tokens/s: 76323.7 | valid tokens/s: 76323.7 | physical tokens/s: 76689.1 | loss tokens/s: 57353.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.404466E-01 | lm-global-supervised-tokens: 1.260722E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.305167E-02 | lm-token-buffer-mib: 2.338399E+02 | global_load_balancing_loss: 9.993363E-01 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11219.210 | params rms: 0.060748 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:46:31.588974 | finish at 2026-10-17 15:52:11 + [2026-10-12 07:09:24] iteration 276/ 2384 | consumed samples: 8832 | elapsed time per iteration (ms): 224411.9 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.44% | FLOPs estimator: workload-aware | logical tokens/s: 74760.7 | valid tokens/s: 74760.7 | physical tokens/s: 76027.3 | loss tokens/s: 60235.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.925493E-01 | lm-global-supervised-tokens: 1.351744E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.551375E-02 | lm-token-buffer-mib: 2.347618E+02 | global_load_balancing_loss: 9.985445E-01 | loss scale: 1.0 | grad norm: 0.477 | params norm: 11219.154 | params rms: 0.060748 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:24:20.358167 | finish at 2026-10-17 18:33:44 + [2026-10-12 07:14:05] iteration 277/ 2384 | consumed samples: 8864 | elapsed time per iteration (ms): 281423.3 | throughput per GPU (TFLOP/s/GPU): 17.4 | MFU 5.57% | FLOPs estimator: workload-aware | logical tokens/s: 59615.5 | valid tokens/s: 59615.5 | physical tokens/s: 60816.6 | loss tokens/s: 43835.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.522072E-01 | lm-global-supervised-tokens: 1.233621E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.043626E-02 | lm-token-buffer-mib: 2.407329E+02 | global_load_balancing_loss: 9.999393E-01 | loss scale: 1.0 | grad norm: 0.381 | params norm: 11219.097 | params rms: 0.060748 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 20:42:38.868544 | finish at 2026-10-19 03:56:44 + [2026-10-12 07:17:49] iteration 278/ 2384 | consumed samples: 8896 | elapsed time per iteration (ms): 223513.5 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.02% | FLOPs estimator: workload-aware | logical tokens/s: 75061.3 | valid tokens/s: 75061.3 | physical tokens/s: 75931.5 | loss tokens/s: 57164.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.923021E-01 | lm-global-supervised-tokens: 1.277714E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.724997E-02 | lm-token-buffer-mib: 2.356483E+02 | global_load_balancing_loss: 1.000625E+00 | loss scale: 1.0 | grad norm: 0.324 | params norm: 11219.027 | params rms: 0.060747 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 10:45:19.490197 | finish at 2026-10-17 18:03:08 + [2026-10-12 07:21:30] iteration 279/ 2384 | consumed samples: 8928 | elapsed time per iteration (ms): 221206.5 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.78% | FLOPs estimator: workload-aware | logical tokens/s: 75844.0 | valid tokens/s: 75844.0 | physical tokens/s: 76416.7 | loss tokens/s: 60473.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.451239E-01 | lm-global-supervised-tokens: 1.337701E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.095069E-02 | lm-token-buffer-mib: 2.336172E+02 | global_load_balancing_loss: 1.001311E+00 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11218.965 | params rms: 0.060747 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 9:20:39.734305 | finish at 2026-10-17 16:42:09 + [2026-10-12 07:25:31] iteration 280/ 2384 | consumed samples: 8960 | elapsed time per iteration (ms): 241219.6 | throughput per GPU (TFLOP/s/GPU): 19.7 | MFU 6.31% | FLOPs estimator: workload-aware | logical tokens/s: 69551.6 | valid tokens/s: 69551.6 | physical tokens/s: 69942.7 | loss tokens/s: 49075.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.759549E-01 | lm-global-supervised-tokens: 1.183800E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.499307E-02 | lm-token-buffer-mib: 2.322530E+02 | global_load_balancing_loss: 1.001139E+00 | loss scale: 1.0 | grad norm: 0.104 | params norm: 11218.912 | params rms: 0.060747 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:58:45.948528 | finish at 2026-10-18 04:24:17 +times across ranks (ms): + save-checkpoint: + rank 0: 24560.59 + rank 1: 24560.60 + rank 2: 24560.59 + rank 3: 24560.56 + rank 4: 24560.55 + rank 5: 24560.59 + rank 6: 24560.59 + rank 7: 24560.59 + rank 8: 24561.95 + rank 9: 24561.99 + rank 10: 24561.97 + rank 11: 24561.95 + rank 12: 24561.96 + rank 13: 24561.95 + rank 14: 24561.96 + rank 15: 24561.95 + rank 16: 24562.21 + rank 17: 24562.19 + rank 18: 24562.19 + rank 19: 24562.19 + rank 20: 24562.17 + rank 21: 24562.18 + rank 22: 24562.18 + rank 23: 24562.18 + rank 24: 24562.81 + rank 25: 24562.85 + rank 26: 24562.83 + rank 27: 24562.82 + rank 28: 24562.86 + rank 29: 24562.80 + rank 30: 24562.80 + rank 31: 24562.81 + rank 32: 24561.64 + rank 33: 24561.65 + rank 34: 24561.66 + rank 35: 24561.77 + rank 36: 24561.74 + rank 37: 24561.67 + rank 38: 24561.60 + rank 39: 24561.52 + rank 40: 24560.64 + rank 41: 24560.62 + rank 42: 24560.62 + rank 43: 24560.64 + rank 44: 24560.61 + rank 45: 24560.48 + rank 46: 24560.59 + rank 47: 24560.60 + rank 48: 24561.62 + rank 49: 24561.50 + rank 50: 24561.45 + rank 51: 24561.53 + rank 52: 24561.42 + rank 53: 24561.42 + rank 54: 24561.42 + rank 55: 24561.49 + rank 56: 24562.40 + rank 57: 24562.44 + rank 58: 24562.44 + rank 59: 24562.42 + rank 60: 24562.43 + rank 61: 24562.41 + rank 62: 24562.39 + rank 63: 24562.38 + [2026-10-12 07:30:01] iteration 281/ 2384 | consumed samples: 8992 | elapsed time per iteration (ms): 245100.8 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.71% | FLOPs estimator: workload-aware | logical tokens/s: 68450.1 | valid tokens/s: 68450.1 | physical tokens/s: 69251.7 | loss tokens/s: 51924.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.583592E-01 | lm-global-supervised-tokens: 1.272670E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 9.029455E-01 | lm-token-buffer-mib: 2.354636E+02 | global_load_balancing_loss: 1.000887E+00 | loss scale: 1.0 | grad norm: 0.158 | params norm: 11218.865 | params rms: 0.060747 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:10:46.892729 | finish at 2026-10-18 06:40:48 + [2026-10-12 07:33:49] iteration 282/ 2384 | consumed samples: 9024 | elapsed time per iteration (ms): 227758.1 | throughput per GPU (TFLOP/s/GPU): 23.9 | MFU 7.65% | FLOPs estimator: workload-aware | logical tokens/s: 73662.1 | valid tokens/s: 73662.1 | physical tokens/s: 74712.8 | loss tokens/s: 59792.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.141860E-01 | lm-global-supervised-tokens: 1.361825E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.017251E-02 | lm-token-buffer-mib: 2.332372E+02 | global_load_balancing_loss: 1.000993E+00 | loss scale: 1.0 | grad norm: 0.155 | params norm: 11218.824 | params rms: 0.060746 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:59:07.607456 | finish at 2026-10-17 20:32:56 + [2026-10-12 07:37:34] iteration 283/ 2384 | consumed samples: 9056 | elapsed time per iteration (ms): 225521.4 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.65% | FLOPs estimator: workload-aware | logical tokens/s: 74392.9 | valid tokens/s: 74392.9 | physical tokens/s: 74871.5 | loss tokens/s: 55196.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.121827E-01 | lm-global-supervised-tokens: 1.244789E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.336093E-02 | lm-token-buffer-mib: 2.339239E+02 | global_load_balancing_loss: 9.998235E-01 | loss scale: 1.0 | grad norm: 0.162 | params norm: 11218.784 | params rms: 0.060746 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:37:00.414762 | finish at 2026-10-17 19:14:35 + [2026-10-12 07:41:16] iteration 284/ 2384 | consumed samples: 9088 | elapsed time per iteration (ms): 221345.3 | throughput per GPU (TFLOP/s/GPU): 24.4 | MFU 7.81% | FLOPs estimator: workload-aware | logical tokens/s: 75796.3 | valid tokens/s: 75796.3 | physical tokens/s: 76895.0 | loss tokens/s: 58828.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.397047E-01 | lm-global-supervised-tokens: 1.302149E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.749711E-02 | lm-token-buffer-mib: 2.330570E+02 | global_load_balancing_loss: 1.001275E+00 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11218.745 | params rms: 0.060746 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 9:07:05.207019 | finish at 2026-10-17 16:48:21 + [2026-10-12 07:44:59] iteration 285/ 2384 | consumed samples: 9120 | elapsed time per iteration (ms): 223236.9 | throughput per GPU (TFLOP/s/GPU): 26.2 | MFU 8.41% | FLOPs estimator: workload-aware | logical tokens/s: 75154.0 | valid tokens/s: 75154.0 | physical tokens/s: 75790.8 | loss tokens/s: 59472.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.697906E-01 | lm-global-supervised-tokens: 1.327641E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.774359E-02 | lm-token-buffer-mib: 2.330679E+02 | global_load_balancing_loss: 1.002509E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11218.707 | params rms: 0.060746 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 10:09:34.341369 | finish at 2026-10-17 17:54:33 + [2026-10-12 07:48:40] iteration 286/ 2384 | consumed samples: 9152 | elapsed time per iteration (ms): 221393.5 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.36% | FLOPs estimator: workload-aware | logical tokens/s: 75780.0 | valid tokens/s: 75779.9 | physical tokens/s: 76429.6 | loss tokens/s: 57795.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.225835E-01 | lm-global-supervised-tokens: 1.279558E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.488539E-02 | lm-token-buffer-mib: 2.331330E+02 | global_load_balancing_loss: 1.000137E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11218.666 | params rms: 0.060745 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 9:01:23.498162 | finish at 2026-10-17 16:50:04 + [2026-10-12 07:52:25] iteration 287/ 2384 | consumed samples: 9184 | elapsed time per iteration (ms): 225090.4 | throughput per GPU (TFLOP/s/GPU): 19.5 | MFU 6.26% | FLOPs estimator: workload-aware | logical tokens/s: 74535.3 | valid tokens/s: 74535.3 | physical tokens/s: 74953.4 | loss tokens/s: 53127.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.232671E-01 | lm-global-supervised-tokens: 1.195847E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.149678E-02 | lm-token-buffer-mib: 2.333488E+02 | global_load_balancing_loss: 1.001787E+00 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11218.625 | params rms: 0.060745 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:06:54.588758 | finish at 2026-10-17 18:59:20 + [2026-10-12 07:56:05] iteration 288/ 2384 | consumed samples: 9216 | elapsed time per iteration (ms): 219547.9 | throughput per GPU (TFLOP/s/GPU): 19.4 | MFU 6.20% | FLOPs estimator: workload-aware | logical tokens/s: 76416.9 | valid tokens/s: 76416.9 | physical tokens/s: 77838.2 | loss tokens/s: 56722.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.544781E-01 | lm-global-supervised-tokens: 1.245338E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.703277E-02 | lm-token-buffer-mib: 2.335863E+02 | global_load_balancing_loss: 9.990307E-01 | loss scale: 1.0 | grad norm: 0.167 | params norm: 11218.583 | params rms: 0.060745 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 7:49:32.380829 | finish at 2026-10-17 15:45:37 + [2026-10-12 07:59:48] iteration 289/ 2384 | consumed samples: 9248 | elapsed time per iteration (ms): 223497.7 | throughput per GPU (TFLOP/s/GPU): 23.5 | MFU 7.54% | FLOPs estimator: workload-aware | logical tokens/s: 75066.6 | valid tokens/s: 75066.6 | physical tokens/s: 75886.5 | loss tokens/s: 54386.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.340811E-01 | lm-global-supervised-tokens: 1.215514E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.972701E-02 | lm-token-buffer-mib: 2.344211E+02 | global_load_balancing_loss: 9.985932E-01 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11218.540 | params rms: 0.060745 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 10:03:47.659972 | finish at 2026-10-17 18:03:36 + [2026-10-12 08:03:50] iteration 290/ 2384 | consumed samples: 9280 | elapsed time per iteration (ms): 241464.9 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.55% | FLOPs estimator: workload-aware | logical tokens/s: 69480.8 | valid tokens/s: 69480.8 | physical tokens/s: 70131.7 | loss tokens/s: 55293.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.279600E-01 | lm-global-supervised-tokens: 1.335131E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.786601E-02 | lm-token-buffer-mib: 2.342965E+02 | global_load_balancing_loss: 9.980164E-01 | loss scale: 1.0 | grad norm: 0.121 | params norm: 11218.494 | params rms: 0.060744 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:27:07.556551 | finish at 2026-10-18 04:30:57 + [2026-10-12 08:07:32] iteration 291/ 2384 | consumed samples: 9312 | elapsed time per iteration (ms): 222256.5 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.47% | FLOPs estimator: workload-aware | logical tokens/s: 75485.6 | valid tokens/s: 75485.6 | physical tokens/s: 76035.0 | loss tokens/s: 59988.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.943010E-01 | lm-global-supervised-tokens: 1.333285E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.576505E-02 | lm-token-buffer-mib: 2.330204E+02 | global_load_balancing_loss: 9.988183E-01 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11218.447 | params rms: 0.060744 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 9:13:02.881958 | finish at 2026-10-17 17:20:35 + [2026-10-12 08:12:07] iteration 292/ 2384 | consumed samples: 9344 | elapsed time per iteration (ms): 275092.3 | throughput per GPU (TFLOP/s/GPU): 16.6 | MFU 5.31% | FLOPs estimator: workload-aware | logical tokens/s: 60987.4 | valid tokens/s: 60987.4 | physical tokens/s: 62109.6 | loss tokens/s: 44949.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.307436E-01 | lm-global-supervised-tokens: 1.236525E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.195083E-02 | lm-token-buffer-mib: 2.382224E+02 | global_load_balancing_loss: 1.001233E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11218.400 | params rms: 0.060744 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 15:51:33.185240 | finish at 2026-10-19 00:03:40 + [2026-10-12 08:15:51] iteration 293/ 2384 | consumed samples: 9376 | elapsed time per iteration (ms): 223545.6 | throughput per GPU (TFLOP/s/GPU): 16.9 | MFU 5.43% | FLOPs estimator: workload-aware | logical tokens/s: 75050.4 | valid tokens/s: 75050.4 | physical tokens/s: 76819.0 | loss tokens/s: 63058.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.219398E-01 | lm-global-supervised-tokens: 1.409635E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.990740E-02 | lm-token-buffer-mib: 2.358413E+02 | global_load_balancing_loss: 1.001259E+00 | loss scale: 1.0 | grad norm: 0.244 | params norm: 11218.354 | params rms: 0.060744 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 9:50:33.935217 | finish at 2026-10-17 18:06:25 + [2026-10-12 08:19:34] iteration 294/ 2384 | consumed samples: 9408 | elapsed time per iteration (ms): 223249.6 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.03% | FLOPs estimator: workload-aware | logical tokens/s: 75149.9 | valid tokens/s: 75149.9 | physical tokens/s: 75718.5 | loss tokens/s: 54104.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.594301E-01 | lm-global-supervised-tokens: 1.207879E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.315774E-02 | lm-token-buffer-mib: 2.337018E+02 | global_load_balancing_loss: 1.003997E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11218.308 | params rms: 0.060743 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 9:36:31.650906 | finish at 2026-10-17 17:56:06 + [2026-10-12 08:23:17] iteration 295/ 2384 | consumed samples: 9440 | elapsed time per iteration (ms): 223128.7 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.21% | FLOPs estimator: workload-aware | logical tokens/s: 75190.6 | valid tokens/s: 75190.6 | physical tokens/s: 75508.6 | loss tokens/s: 56760.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.276405E-01 | lm-global-supervised-tokens: 1.266486E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.746167E-02 | lm-token-buffer-mib: 2.334030E+02 | global_load_balancing_loss: 1.003305E+00 | loss scale: 1.0 | grad norm: 0.113 | params norm: 11218.258 | params rms: 0.060743 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 9:28:35.868283 | finish at 2026-10-17 17:51:53 + [2026-10-12 08:27:03] iteration 296/ 2384 | consumed samples: 9472 | elapsed time per iteration (ms): 226219.5 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.11% | FLOPs estimator: workload-aware | logical tokens/s: 74163.2 | valid tokens/s: 74163.2 | physical tokens/s: 75157.9 | loss tokens/s: 57961.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.806692E-01 | lm-global-supervised-tokens: 1.311210E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.129389E-02 | lm-token-buffer-mib: 2.364314E+02 | global_load_balancing_loss: 1.001496E+00 | loss scale: 1.0 | grad norm: 0.250 | params norm: 11218.208 | params rms: 0.060743 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:12:26.290249 | finish at 2026-10-17 19:39:30 + [2026-10-12 08:30:51] iteration 297/ 2384 | consumed samples: 9504 | elapsed time per iteration (ms): 227235.5 | throughput per GPU (TFLOP/s/GPU): 18.3 | MFU 5.86% | FLOPs estimator: workload-aware | logical tokens/s: 73831.7 | valid tokens/s: 73831.7 | physical tokens/s: 74995.3 | loss tokens/s: 55389.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.560395E-01 | lm-global-supervised-tokens: 1.258644E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.350246E-02 | lm-token-buffer-mib: 2.353806E+02 | global_load_balancing_loss: 1.000294E+00 | loss scale: 1.0 | grad norm: 0.125 | params norm: 11218.155 | params rms: 0.060743 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:44:00.545427 | finish at 2026-10-17 20:14:51 + [2026-10-12 08:34:39] iteration 298/ 2384 | consumed samples: 9536 | elapsed time per iteration (ms): 228042.5 | throughput per GPU (TFLOP/s/GPU): 25.2 | MFU 8.09% | FLOPs estimator: workload-aware | logical tokens/s: 73570.5 | valid tokens/s: 73570.5 | physical tokens/s: 74203.1 | loss tokens/s: 60866.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.439851E-01 | lm-global-supervised-tokens: 1.388010E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.304335E-02 | lm-token-buffer-mib: 2.341622E+02 | global_load_balancing_loss: 9.994999E-01 | loss scale: 1.0 | grad norm: 0.112 | params norm: 11218.102 | params rms: 0.060742 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:08:16.668468 | finish at 2026-10-17 20:42:55 + [2026-10-12 08:38:21] iteration 299/ 2384 | consumed samples: 9568 | elapsed time per iteration (ms): 222175.6 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.03% | FLOPs estimator: workload-aware | logical tokens/s: 75513.3 | valid tokens/s: 75513.3 | physical tokens/s: 76052.0 | loss tokens/s: 57204.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.190895E-01 | lm-global-supervised-tokens: 1.270943E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.284477E-02 | lm-token-buffer-mib: 2.346425E+02 | global_load_balancing_loss: 1.001348E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11218.051 | params rms: 0.060742 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:40:36.229008 | finish at 2026-10-17 17:18:57 + [2026-10-12 08:42:24] iteration 300/ 2384 | consumed samples: 9600 | elapsed time per iteration (ms): 243639.0 | throughput per GPU (TFLOP/s/GPU): 25.9 | MFU 8.30% | FLOPs estimator: workload-aware | logical tokens/s: 68860.7 | valid tokens/s: 68860.7 | physical tokens/s: 69491.1 | loss tokens/s: 52628.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.731886E-01 | lm-global-supervised-tokens: 1.282233E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.063171E-02 | lm-token-buffer-mib: 2.325250E+02 | global_load_balancing_loss: 9.982749E-01 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11218.000 | params rms: 0.060742 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:02:23.755868 | finish at 2026-10-18 05:44:48 + [2026-10-12 08:46:04] iteration 301/ 2384 | consumed samples: 9632 | elapsed time per iteration (ms): 219875.0 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.41% | FLOPs estimator: workload-aware | logical tokens/s: 76303.3 | valid tokens/s: 76303.3 | physical tokens/s: 76928.1 | loss tokens/s: 59392.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.988562E-01 | lm-global-supervised-tokens: 1.305893E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.919703E-02 | lm-token-buffer-mib: 2.336985E+02 | global_load_balancing_loss: 9.991685E-01 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11217.948 | params rms: 0.060742 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 7:13:19.627980 | finish at 2026-10-17 15:59:24 + [2026-10-12 08:49:47] iteration 302/ 2384 | consumed samples: 9664 | elapsed time per iteration (ms): 222714.0 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.21% | FLOPs estimator: workload-aware | logical tokens/s: 75330.6 | valid tokens/s: 75330.6 | physical tokens/s: 76307.5 | loss tokens/s: 57845.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.268192E-01 | lm-global-supervised-tokens: 1.288300E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.297602E-02 | lm-token-buffer-mib: 2.338423E+02 | global_load_balancing_loss: 9.991417E-01 | loss scale: 1.0 | grad norm: 0.115 | params norm: 11217.898 | params rms: 0.060741 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:48:10.644768 | finish at 2026-10-17 17:37:58 + [2026-10-12 08:53:26] iteration 303/ 2384 | consumed samples: 9696 | elapsed time per iteration (ms): 218942.8 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.59% | FLOPs estimator: workload-aware | logical tokens/s: 76628.1 | valid tokens/s: 76628.1 | physical tokens/s: 77125.3 | loss tokens/s: 58476.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.562861E-01 | lm-global-supervised-tokens: 1.280300E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.292962E-02 | lm-token-buffer-mib: 2.330835E+02 | global_load_balancing_loss: 1.000810E+00 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11217.850 | params rms: 0.060741 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:33:39.888998 | finish at 2026-10-17 15:27:06 + [2026-10-12 08:57:09] iteration 304/ 2384 | consumed samples: 9728 | elapsed time per iteration (ms): 222609.7 | throughput per GPU (TFLOP/s/GPU): 24.6 | MFU 7.89% | FLOPs estimator: workload-aware | logical tokens/s: 75366.0 | valid tokens/s: 75366.0 | physical tokens/s: 75685.2 | loss tokens/s: 57194.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.829112E-01 | lm-global-supervised-tokens: 1.273194E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.848867E-02 | lm-token-buffer-mib: 2.330169E+02 | global_load_balancing_loss: 1.002866E+00 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11217.799 | params rms: 0.060741 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:37:08.261223 | finish at 2026-10-17 17:34:17 + [2026-10-12 09:00:50] iteration 305/ 2384 | consumed samples: 9760 | elapsed time per iteration (ms): 221188.5 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.79% | FLOPs estimator: workload-aware | logical tokens/s: 75850.3 | valid tokens/s: 75850.3 | physical tokens/s: 76539.2 | loss tokens/s: 57904.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.514814E-01 | lm-global-supervised-tokens: 1.280786E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.263595E-02 | lm-token-buffer-mib: 2.340012E+02 | global_load_balancing_loss: 1.001108E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11217.740 | params rms: 0.060740 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 7:44:10.925551 | finish at 2026-10-17 16:45:01 + [2026-10-12 09:04:33] iteration 306/ 2384 | consumed samples: 9792 | elapsed time per iteration (ms): 223510.9 | throughput per GPU (TFLOP/s/GPU): 18.7 | MFU 6.01% | FLOPs estimator: workload-aware | logical tokens/s: 75062.1 | valid tokens/s: 75062.1 | physical tokens/s: 76112.5 | loss tokens/s: 61948.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.896793E-01 | lm-global-supervised-tokens: 1.384612E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.109336E-02 | lm-token-buffer-mib: 2.361244E+02 | global_load_balancing_loss: 1.001217E+00 | loss scale: 1.0 | grad norm: 0.123 | params norm: 11217.680 | params rms: 0.060740 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 9:00:55.693841 | finish at 2026-10-17 18:05:29 + [2026-10-12 09:08:20] iteration 307/ 2384 | consumed samples: 9824 | elapsed time per iteration (ms): 226485.9 | throughput per GPU (TFLOP/s/GPU): 24.5 | MFU 7.84% | FLOPs estimator: workload-aware | logical tokens/s: 74075.9 | valid tokens/s: 74075.9 | physical tokens/s: 74480.0 | loss tokens/s: 56976.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.375826E-01 | lm-global-supervised-tokens: 1.290426E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.850001E-02 | lm-token-buffer-mib: 2.336607E+02 | global_load_balancing_loss: 1.001928E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11217.625 | params rms: 0.060740 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 10:40:11.197803 | finish at 2026-10-17 19:48:31 + [2026-10-12 09:12:02] iteration 308/ 2384 | consumed samples: 9856 | elapsed time per iteration (ms): 222253.3 | throughput per GPU (TFLOP/s/GPU): 19.0 | MFU 6.09% | FLOPs estimator: workload-aware | logical tokens/s: 75486.8 | valid tokens/s: 75486.8 | physical tokens/s: 76153.8 | loss tokens/s: 58579.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.122817E-01 | lm-global-supervised-tokens: 1.301954E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.571169E-02 | lm-token-buffer-mib: 2.347978E+02 | global_load_balancing_loss: 9.999895E-01 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11217.571 | params rms: 0.060739 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:09:57.928408 | finish at 2026-10-17 17:22:00 + [2026-10-12 09:15:45] iteration 309/ 2384 | consumed samples: 9888 | elapsed time per iteration (ms): 222971.1 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.97% | FLOPs estimator: workload-aware | logical tokens/s: 75243.9 | valid tokens/s: 75243.9 | physical tokens/s: 76511.1 | loss tokens/s: 56589.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.380052E-01 | lm-global-supervised-tokens: 1.261785E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.853641E-02 | lm-token-buffer-mib: 2.356678E+02 | global_load_balancing_loss: 9.994983E-01 | loss scale: 1.0 | grad norm: 0.190 | params norm: 11217.513 | params rms: 0.060739 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:31:05.101457 | finish at 2026-10-17 17:46:50 + [2026-10-12 09:19:53] iteration 310/ 2384 | consumed samples: 9920 | elapsed time per iteration (ms): 247547.8 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.54% | FLOPs estimator: workload-aware | logical tokens/s: 67773.6 | valid tokens/s: 67773.6 | physical tokens/s: 68631.2 | loss tokens/s: 53511.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.969408E-01 | lm-global-supervised-tokens: 1.324663E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.675149E-02 | lm-token-buffer-mib: 2.349331E+02 | global_load_balancing_loss: 1.001826E+00 | loss scale: 1.0 | grad norm: 0.205 | params norm: 11217.454 | params rms: 0.060739 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:36:54.116565 | finish at 2026-10-18 07:56:47 + [2026-10-12 09:23:36] iteration 311/ 2384 | consumed samples: 9952 | elapsed time per iteration (ms): 223472.2 | throughput per GPU (TFLOP/s/GPU): 26.9 | MFU 8.62% | FLOPs estimator: workload-aware | logical tokens/s: 75075.0 | valid tokens/s: 75075.0 | physical tokens/s: 75640.2 | loss tokens/s: 55094.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.013293E-01 | lm-global-supervised-tokens: 1.231200E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.512752E-02 | lm-token-buffer-mib: 2.343840E+02 | global_load_balancing_loss: 1.000428E+00 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11217.397 | params rms: 0.060739 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:40:57.882784 | finish at 2026-10-17 18:04:34 + [2026-10-12 09:27:20] iteration 312/ 2384 | consumed samples: 9984 | elapsed time per iteration (ms): 223657.3 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.57% | FLOPs estimator: workload-aware | logical tokens/s: 75013.0 | valid tokens/s: 75013.0 | physical tokens/s: 75720.4 | loss tokens/s: 58232.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.502693E-01 | lm-global-supervised-tokens: 1.302414E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.647645E-02 | lm-token-buffer-mib: 2.327580E+02 | global_load_balancing_loss: 1.001010E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11217.340 | params rms: 0.060738 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:43:37.948809 | finish at 2026-10-17 18:10:58 + [2026-10-12 09:31:04] iteration 313/ 2384 | consumed samples: 10016 | elapsed time per iteration (ms): 224021.6 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.04% | FLOPs estimator: workload-aware | logical tokens/s: 74890.9 | valid tokens/s: 74890.9 | physical tokens/s: 75969.2 | loss tokens/s: 56559.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.114722E-01 | lm-global-supervised-tokens: 1.267046E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.619019E-02 | lm-token-buffer-mib: 2.355015E+02 | global_load_balancing_loss: 1.000207E+00 | loss scale: 1.0 | grad norm: 0.133 | params norm: 11217.286 | params rms: 0.060738 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:52:28.727691 | finish at 2026-10-17 18:23:32 + [2026-10-12 09:34:45] iteration 314/ 2384 | consumed samples: 10048 | elapsed time per iteration (ms): 221045.5 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.15% | FLOPs estimator: workload-aware | logical tokens/s: 75899.3 | valid tokens/s: 75899.3 | physical tokens/s: 76524.5 | loss tokens/s: 62778.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.768421E-01 | lm-global-supervised-tokens: 1.387689E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.607935E-02 | lm-token-buffer-mib: 2.338562E+02 | global_load_balancing_loss: 9.997768E-01 | loss scale: 1.0 | grad norm: 0.198 | params norm: 11217.231 | params rms: 0.060738 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 7:06:04.171758 | finish at 2026-10-17 16:40:49 + [2026-10-12 09:38:27] iteration 315/ 2384 | consumed samples: 10080 | elapsed time per iteration (ms): 222046.5 | throughput per GPU (TFLOP/s/GPU): 23.5 | MFU 7.55% | FLOPs estimator: workload-aware | logical tokens/s: 75557.1 | valid tokens/s: 75557.1 | physical tokens/s: 76279.8 | loss tokens/s: 55798.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.495719E-01 | lm-global-supervised-tokens: 1.238981E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.208408E-02 | lm-token-buffer-mib: 2.336317E+02 | global_load_balancing_loss: 1.000542E+00 | loss scale: 1.0 | grad norm: 0.139 | params norm: 11217.174 | params rms: 0.060737 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 7:36:54.117175 | finish at 2026-10-17 17:15:21 + [2026-10-12 09:42:08] iteration 316/ 2384 | consumed samples: 10112 | elapsed time per iteration (ms): 221382.7 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.52% | FLOPs estimator: workload-aware | logical tokens/s: 75783.7 | valid tokens/s: 75783.7 | physical tokens/s: 76475.9 | loss tokens/s: 58704.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.253346E-01 | lm-global-supervised-tokens: 1.299608E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.548048E-02 | lm-token-buffer-mib: 2.344858E+02 | global_load_balancing_loss: 9.998728E-01 | loss scale: 1.0 | grad norm: 0.231 | params norm: 11217.116 | params rms: 0.060737 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 7:10:19.386552 | finish at 2026-10-17 16:52:28 + [2026-10-12 09:45:49] iteration 317/ 2384 | consumed samples: 10144 | elapsed time per iteration (ms): 220911.2 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.83% | FLOPs estimator: workload-aware | logical tokens/s: 75945.3 | valid tokens/s: 75945.3 | physical tokens/s: 76831.7 | loss tokens/s: 56717.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.057788E-01 | lm-global-supervised-tokens: 1.252949E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.677446E-02 | lm-token-buffer-mib: 2.347129E+02 | global_load_balancing_loss: 9.994874E-01 | loss scale: 1.0 | grad norm: 0.148 | params norm: 11217.057 | params rms: 0.060737 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:50:23.466266 | finish at 2026-10-17 16:36:13 + [2026-10-12 09:49:31] iteration 318/ 2384 | consumed samples: 10176 | elapsed time per iteration (ms): 221517.4 | throughput per GPU (TFLOP/s/GPU): 19.9 | MFU 6.36% | FLOPs estimator: workload-aware | logical tokens/s: 75737.5 | valid tokens/s: 75737.5 | physical tokens/s: 76407.7 | loss tokens/s: 58422.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.950673E-01 | lm-global-supervised-tokens: 1.294155E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.280205E-02 | lm-token-buffer-mib: 2.348557E+02 | global_load_balancing_loss: 1.001084E+00 | loss scale: 1.0 | grad norm: 0.107 | params norm: 11216.996 | params rms: 0.060736 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 7:07:34.846984 | finish at 2026-10-17 16:57:05 + [2026-10-12 09:53:12] iteration 319/ 2384 | consumed samples: 10208 | elapsed time per iteration (ms): 221370.4 | throughput per GPU (TFLOP/s/GPU): 28.0 | MFU 8.99% | FLOPs estimator: workload-aware | logical tokens/s: 75787.7 | valid tokens/s: 75787.7 | physical tokens/s: 76415.6 | loss tokens/s: 56164.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 9.268942E-01 | lm-global-supervised-tokens: 1.243309E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.696726E-02 | lm-token-buffer-mib: 2.324145E+02 | global_load_balancing_loss: 1.002020E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11216.934 | params rms: 0.060736 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:58:49.951228 | finish at 2026-10-17 16:52:02 + [2026-10-12 09:57:12] iteration 320/ 2384 | consumed samples: 10240 | elapsed time per iteration (ms): 240348.2 | throughput per GPU (TFLOP/s/GPU): 24.9 | MFU 7.97% | FLOPs estimator: workload-aware | logical tokens/s: 69803.7 | valid tokens/s: 69803.7 | physical tokens/s: 70103.6 | loss tokens/s: 50463.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.464275E-01 | lm-global-supervised-tokens: 1.212876E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.744806E-02 | lm-token-buffer-mib: 2.329696E+02 | global_load_balancing_loss: 1.001921E+00 | loss scale: 1.0 | grad norm: 0.131 | params norm: 11216.874 | params rms: 0.060736 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:47:58.719910 | finish at 2026-10-18 03:45:11 +times across ranks (ms): + save-checkpoint: + rank 0: 25750.11 + rank 1: 25750.07 + rank 2: 25750.08 + rank 3: 25750.08 + rank 4: 25750.08 + rank 5: 25750.08 + rank 6: 25750.07 + rank 7: 25750.13 + rank 8: 25750.05 + rank 9: 25750.08 + rank 10: 25750.03 + rank 11: 25749.98 + rank 12: 25749.99 + rank 13: 25749.97 + rank 14: 25750.00 + rank 15: 25749.97 + rank 16: 25750.07 + rank 17: 25750.09 + rank 18: 25750.12 + rank 19: 25750.14 + rank 20: 25750.07 + rank 21: 25750.07 + rank 22: 25750.13 + rank 23: 25750.08 + rank 24: 25750.20 + rank 25: 25750.20 + rank 26: 25750.11 + rank 27: 25750.14 + rank 28: 25749.92 + rank 29: 25750.21 + rank 30: 25750.23 + rank 31: 25750.14 + rank 32: 25750.47 + rank 33: 25750.42 + rank 34: 25750.45 + rank 35: 25750.37 + rank 36: 25750.39 + rank 37: 25750.42 + rank 38: 25750.39 + rank 39: 25750.39 + rank 40: 25750.14 + rank 41: 25750.13 + rank 42: 25750.12 + rank 43: 25750.13 + rank 44: 25750.12 + rank 45: 25750.11 + rank 46: 25750.10 + rank 47: 25750.13 + rank 48: 25750.81 + rank 49: 25750.91 + rank 50: 25750.85 + rank 51: 25751.05 + rank 52: 25750.83 + rank 53: 25750.83 + rank 54: 25750.83 + rank 55: 25750.83 + rank 56: 25750.23 + rank 57: 25750.29 + rank 58: 25750.29 + rank 59: 25750.29 + rank 60: 25750.28 + rank 61: 25750.23 + rank 62: 25750.25 + rank 63: 25750.21 + [2026-10-12 10:01:22] iteration 321/ 2384 | consumed samples: 10272 | elapsed time per iteration (ms): 223589.3 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.91% | FLOPs estimator: workload-aware | logical tokens/s: 75035.7 | valid tokens/s: 75035.7 | physical tokens/s: 75682.8 | loss tokens/s: 55888.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.246952E-01 | lm-global-supervised-tokens: 1.249607E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 6.439856E-02 | lm-token-buffer-mib: 2.331747E+02 | global_load_balancing_loss: 1.000327E+00 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11216.812 | params rms: 0.060735 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:07:44.733599 | finish at 2026-10-17 18:09:06 + [2026-10-12 10:05:09] iteration 322/ 2384 | consumed samples: 10304 | elapsed time per iteration (ms): 227657.8 | throughput per GPU (TFLOP/s/GPU): 24.6 | MFU 7.89% | FLOPs estimator: workload-aware | logical tokens/s: 73694.8 | valid tokens/s: 73694.8 | physical tokens/s: 74154.0 | loss tokens/s: 59975.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.792954E-01 | lm-global-supervised-tokens: 1.365388E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.985421E-02 | lm-token-buffer-mib: 2.335993E+02 | global_load_balancing_loss: 1.000519E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11216.754 | params rms: 0.060735 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 10:23:50.462175 | finish at 2026-10-17 20:29:00 + [2026-10-12 10:08:50] iteration 323/ 2384 | consumed samples: 10336 | elapsed time per iteration (ms): 220504.4 | throughput per GPU (TFLOP/s/GPU): 27.7 | MFU 8.88% | FLOPs estimator: workload-aware | logical tokens/s: 76085.4 | valid tokens/s: 76085.4 | physical tokens/s: 76719.8 | loss tokens/s: 61638.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.981634E-01 | lm-global-supervised-tokens: 1.359162E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.671519E-02 | lm-token-buffer-mib: 2.332242E+02 | global_load_balancing_loss: 9.993498E-01 | loss scale: 1.0 | grad norm: 0.119 | params norm: 11216.697 | params rms: 0.060735 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:14:19.666215 | finish at 2026-10-17 16:23:09 + [2026-10-12 10:12:32] iteration 324/ 2384 | consumed samples: 10368 | elapsed time per iteration (ms): 221729.6 | throughput per GPU (TFLOP/s/GPU): 19.4 | MFU 6.21% | FLOPs estimator: workload-aware | logical tokens/s: 75665.2 | valid tokens/s: 75665.2 | physical tokens/s: 76182.7 | loss tokens/s: 56433.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.724797E-01 | lm-global-supervised-tokens: 1.251293E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.937678E-02 | lm-token-buffer-mib: 2.340467E+02 | global_load_balancing_loss: 9.992922E-01 | loss scale: 1.0 | grad norm: 0.117 | params norm: 11216.642 | params rms: 0.060734 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:52:43.039751 | finish at 2026-10-17 17:05:15 + [2026-10-12 10:16:12] iteration 325/ 2384 | consumed samples: 10400 | elapsed time per iteration (ms): 220368.7 | throughput per GPU (TFLOP/s/GPU): 24.1 | MFU 7.71% | FLOPs estimator: workload-aware | logical tokens/s: 76132.3 | valid tokens/s: 76132.3 | physical tokens/s: 76542.3 | loss tokens/s: 56947.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.704823E-01 | lm-global-supervised-tokens: 1.254935E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.340093E-02 | lm-token-buffer-mib: 2.330190E+02 | global_load_balancing_loss: 1.002173E+00 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11216.591 | params rms: 0.060734 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:02:19.159738 | finish at 2026-10-17 16:18:31 + [2026-10-12 10:19:55] iteration 326/ 2384 | consumed samples: 10432 | elapsed time per iteration (ms): 222963.2 | throughput per GPU (TFLOP/s/GPU): 23.4 | MFU 7.51% | FLOPs estimator: workload-aware | logical tokens/s: 75246.5 | valid tokens/s: 75246.5 | physical tokens/s: 75760.7 | loss tokens/s: 61033.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.321286E-01 | lm-global-supervised-tokens: 1.360829E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.752356E-02 | lm-token-buffer-mib: 2.339334E+02 | global_load_balancing_loss: 1.000915E+00 | loss scale: 1.0 | grad norm: 0.133 | params norm: 11216.541 | params rms: 0.060734 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 7:27:38.199550 | finish at 2026-10-17 17:47:33 + [2026-10-12 10:23:43] iteration 327/ 2384 | consumed samples: 10464 | elapsed time per iteration (ms): 227856.9 | throughput per GPU (TFLOP/s/GPU): 26.8 | MFU 8.58% | FLOPs estimator: workload-aware | logical tokens/s: 73630.3 | valid tokens/s: 73630.3 | physical tokens/s: 74447.0 | loss tokens/s: 57873.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.053601E-01 | lm-global-supervised-tokens: 1.318696E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.649303E-02 | lm-token-buffer-mib: 2.335292E+02 | global_load_balancing_loss: 9.990826E-01 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11216.491 | params rms: 0.060734 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 10:11:41.630991 | finish at 2026-10-17 20:35:24 + [2026-10-12 10:27:26] iteration 328/ 2384 | consumed samples: 10496 | elapsed time per iteration (ms): 223008.8 | throughput per GPU (TFLOP/s/GPU): 24.4 | MFU 7.82% | FLOPs estimator: workload-aware | logical tokens/s: 75231.0 | valid tokens/s: 75231.0 | physical tokens/s: 75827.8 | loss tokens/s: 61570.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.846053E-01 | lm-global-supervised-tokens: 1.373080E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.453076E-02 | lm-token-buffer-mib: 2.347976E+02 | global_load_balancing_loss: 1.001653E+00 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11216.441 | params rms: 0.060733 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 7:21:46.181585 | finish at 2026-10-17 17:49:12 + [2026-10-12 10:31:07] iteration 329/ 2384 | consumed samples: 10528 | elapsed time per iteration (ms): 221055.0 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.79% | FLOPs estimator: workload-aware | logical tokens/s: 75895.9 | valid tokens/s: 75895.9 | physical tokens/s: 76445.3 | loss tokens/s: 53231.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.842802E-01 | lm-global-supervised-tokens: 1.176699E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.729044E-02 | lm-token-buffer-mib: 2.339424E+02 | global_load_balancing_loss: 1.000646E+00 | loss scale: 1.0 | grad norm: 0.176 | params norm: 11216.392 | params rms: 0.060733 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:11:07.963403 | finish at 2026-10-17 16:42:15 + [2026-10-12 10:35:15] iteration 330/ 2384 | consumed samples: 10560 | elapsed time per iteration (ms): 248683.1 | throughput per GPU (TFLOP/s/GPU): 24.2 | MFU 7.77% | FLOPs estimator: workload-aware | logical tokens/s: 67464.1 | valid tokens/s: 67464.1 | physical tokens/s: 68073.2 | loss tokens/s: 52183.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.424689E-01 | lm-global-supervised-tokens: 1.297722E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.732664E-02 | lm-token-buffer-mib: 2.334445E+02 | global_load_balancing_loss: 1.000647E+00 | loss scale: 1.0 | grad norm: 0.165 | params norm: 11216.346 | params rms: 0.060733 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:53:15.026155 | finish at 2026-10-18 08:28:31 + [2026-10-12 10:39:00] iteration 331/ 2384 | consumed samples: 10592 | elapsed time per iteration (ms): 224707.1 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.22% | FLOPs estimator: workload-aware | logical tokens/s: 74662.4 | valid tokens/s: 74662.4 | physical tokens/s: 75562.9 | loss tokens/s: 54331.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.099788E-01 | lm-global-supervised-tokens: 1.220860E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.740181E-02 | lm-token-buffer-mib: 2.332040E+02 | global_load_balancing_loss: 1.000265E+00 | loss scale: 1.0 | grad norm: 0.122 | params norm: 11216.300 | params rms: 0.060733 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:08:43.710877 | finish at 2026-10-17 18:47:44 + [2026-10-12 10:42:41] iteration 332/ 2384 | consumed samples: 10624 | elapsed time per iteration (ms): 221134.4 | throughput per GPU (TFLOP/s/GPU): 23.9 | MFU 7.65% | FLOPs estimator: workload-aware | logical tokens/s: 75868.7 | valid tokens/s: 75868.7 | physical tokens/s: 76439.9 | loss tokens/s: 53652.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.483653E-01 | lm-global-supervised-tokens: 1.186432E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.357440E-02 | lm-token-buffer-mib: 2.340215E+02 | global_load_balancing_loss: 1.001064E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11216.253 | params rms: 0.060732 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:02:47.840435 | finish at 2026-10-17 16:45:29 + [2026-10-12 10:46:23] iteration 333/ 2384 | consumed samples: 10656 | elapsed time per iteration (ms): 221756.7 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.23% | FLOPs estimator: workload-aware | logical tokens/s: 75655.9 | valid tokens/s: 75655.9 | physical tokens/s: 75935.0 | loss tokens/s: 52187.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.773472E-01 | lm-global-supervised-tokens: 1.157282E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.487178E-02 | lm-token-buffer-mib: 2.329706E+02 | global_load_balancing_loss: 1.001186E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11216.208 | params rms: 0.060732 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:20:22.990802 | finish at 2026-10-17 17:06:46 + [2026-10-12 10:50:05] iteration 334/ 2384 | consumed samples: 10688 | elapsed time per iteration (ms): 221970.3 | throughput per GPU (TFLOP/s/GPU): 19.7 | MFU 6.30% | FLOPs estimator: workload-aware | logical tokens/s: 75583.2 | valid tokens/s: 75583.2 | physical tokens/s: 75942.7 | loss tokens/s: 56764.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.744206E-01 | lm-global-supervised-tokens: 1.259992E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.280350E-02 | lm-token-buffer-mib: 2.330455E+02 | global_load_balancing_loss: 1.001116E+00 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11216.163 | params rms: 0.060732 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:23:59.120293 | finish at 2026-10-17 17:14:04 + [2026-10-12 10:53:50] iteration 335/ 2384 | consumed samples: 10720 | elapsed time per iteration (ms): 224470.9 | throughput per GPU (TFLOP/s/GPU): 24.0 | MFU 7.71% | FLOPs estimator: workload-aware | logical tokens/s: 74741.1 | valid tokens/s: 74741.1 | physical tokens/s: 75438.9 | loss tokens/s: 50647.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.449394E-01 | lm-global-supervised-tokens: 1.136881E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.469065E-02 | lm-token-buffer-mib: 2.337703E+02 | global_load_balancing_loss: 1.000582E+00 | loss scale: 1.0 | grad norm: 0.425 | params norm: 11216.121 | params rms: 0.060732 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 7:45:40.786287 | finish at 2026-10-17 18:39:30 + [2026-10-12 10:57:35] iteration 336/ 2384 | consumed samples: 10752 | elapsed time per iteration (ms): 225351.8 | throughput per GPU (TFLOP/s/GPU): 24.0 | MFU 7.68% | FLOPs estimator: workload-aware | logical tokens/s: 74448.9 | valid tokens/s: 74448.9 | physical tokens/s: 74876.1 | loss tokens/s: 56993.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.047768E-01 | lm-global-supervised-tokens: 1.284358E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.571272E-02 | lm-token-buffer-mib: 2.336034E+02 | global_load_balancing_loss: 1.000391E+00 | loss scale: 1.0 | grad norm: 0.286 | params norm: 11216.079 | params rms: 0.060731 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:12:00.401855 | finish at 2026-10-17 19:09:35 + [2026-10-12 11:01:17] iteration 337/ 2384 | consumed samples: 10784 | elapsed time per iteration (ms): 222432.4 | throughput per GPU (TFLOP/s/GPU): 24.6 | MFU 7.87% | FLOPs estimator: workload-aware | logical tokens/s: 75425.9 | valid tokens/s: 75425.8 | physical tokens/s: 76163.6 | loss tokens/s: 57540.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.754597E-01 | lm-global-supervised-tokens: 1.279885E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.800762E-02 | lm-token-buffer-mib: 2.340045E+02 | global_load_balancing_loss: 1.001455E+00 | loss scale: 1.0 | grad norm: 0.325 | params norm: 11216.034 | params rms: 0.060731 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:28:39.194518 | finish at 2026-10-17 17:29:57 + [2026-10-12 11:04:59] iteration 338/ 2384 | consumed samples: 10816 | elapsed time per iteration (ms): 221300.1 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.32% | FLOPs estimator: workload-aware | logical tokens/s: 75811.8 | valid tokens/s: 75811.8 | physical tokens/s: 76375.4 | loss tokens/s: 61096.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.461739E-01 | lm-global-supervised-tokens: 1.352056E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.910433E-02 | lm-token-buffer-mib: 2.345024E+02 | global_load_balancing_loss: 1.000418E+00 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11215.989 | params rms: 0.060731 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:46:20.103805 | finish at 2026-10-17 16:51:19 + [2026-10-12 11:08:39] iteration 339/ 2384 | consumed samples: 10848 | elapsed time per iteration (ms): 220276.3 | throughput per GPU (TFLOP/s/GPU): 18.8 | MFU 6.02% | FLOPs estimator: workload-aware | logical tokens/s: 76164.4 | valid tokens/s: 76164.4 | physical tokens/s: 76968.1 | loss tokens/s: 59797.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.739499E-01 | lm-global-supervised-tokens: 1.317187E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.053146E-02 | lm-token-buffer-mib: 2.348017E+02 | global_load_balancing_loss: 1.000296E+00 | loss scale: 1.0 | grad norm: 0.374 | params norm: 11215.944 | params rms: 0.060731 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:07:44.951981 | finish at 2026-10-17 16:16:24 + [2026-10-12 11:12:41] iteration 340/ 2384 | consumed samples: 10880 | elapsed time per iteration (ms): 242229.8 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.35% | FLOPs estimator: workload-aware | logical tokens/s: 69261.5 | valid tokens/s: 69261.5 | physical tokens/s: 69679.0 | loss tokens/s: 52213.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.268649E-01 | lm-global-supervised-tokens: 1.264775E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.488287E-02 | lm-token-buffer-mib: 2.330031E+02 | global_load_balancing_loss: 1.000797E+00 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11215.898 | params rms: 0.060730 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:31:57.732126 | finish at 2026-10-18 04:44:39 + [2026-10-12 11:16:23] iteration 341/ 2384 | consumed samples: 10912 | elapsed time per iteration (ms): 222183.6 | throughput per GPU (TFLOP/s/GPU): 24.8 | MFU 7.95% | FLOPs estimator: workload-aware | logical tokens/s: 75510.4 | valid tokens/s: 75510.4 | physical tokens/s: 76244.3 | loss tokens/s: 59381.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.435949E-01 | lm-global-supervised-tokens: 1.319357E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.654151E-02 | lm-token-buffer-mib: 2.362055E+02 | global_load_balancing_loss: 9.991992E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11215.852 | params rms: 0.060730 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:05:21.124408 | finish at 2026-10-17 17:21:44 + [2026-10-12 11:20:07] iteration 342/ 2384 | consumed samples: 10944 | elapsed time per iteration (ms): 223363.6 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.78% | FLOPs estimator: workload-aware | logical tokens/s: 75111.5 | valid tokens/s: 75111.5 | physical tokens/s: 75835.2 | loss tokens/s: 52596.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.135239E-01 | lm-global-supervised-tokens: 1.174809E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.812387E-02 | lm-token-buffer-mib: 2.357536E+02 | global_load_balancing_loss: 9.995120E-01 | loss scale: 1.0 | grad norm: 0.131 | params norm: 11215.807 | params rms: 0.060730 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:41:48.375809 | finish at 2026-10-17 18:01:55 + [2026-10-12 11:23:48] iteration 343/ 2384 | consumed samples: 10976 | elapsed time per iteration (ms): 221215.2 | throughput per GPU (TFLOP/s/GPU): 18.9 | MFU 6.07% | FLOPs estimator: workload-aware | logical tokens/s: 75841.1 | valid tokens/s: 75841.1 | physical tokens/s: 76741.8 | loss tokens/s: 57784.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.429739E-01 | lm-global-supervised-tokens: 1.278290E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.494522E-02 | lm-token-buffer-mib: 2.333981E+02 | global_load_balancing_loss: 1.001796E+00 | loss scale: 1.0 | grad norm: 0.095 | params norm: 11215.762 | params rms: 0.060730 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:25:00.151074 | finish at 2026-10-17 16:48:48 + [2026-10-12 11:27:29] iteration 344/ 2384 | consumed samples: 11008 | elapsed time per iteration (ms): 221503.4 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.77% | FLOPs estimator: workload-aware | logical tokens/s: 75742.4 | valid tokens/s: 75742.3 | physical tokens/s: 76304.2 | loss tokens/s: 60940.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.884207E-01 | lm-global-supervised-tokens: 1.349852E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.038342E-02 | lm-token-buffer-mib: 2.342478E+02 | global_load_balancing_loss: 1.001737E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11215.716 | params rms: 0.060729 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:31:06.929369 | finish at 2026-10-17 16:58:36 + [2026-10-12 11:31:13] iteration 345/ 2384 | consumed samples: 11040 | elapsed time per iteration (ms): 223153.6 | throughput per GPU (TFLOP/s/GPU): 24.0 | MFU 7.68% | FLOPs estimator: workload-aware | logical tokens/s: 75182.2 | valid tokens/s: 75182.2 | physical tokens/s: 75643.5 | loss tokens/s: 52219.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.136949E-01 | lm-global-supervised-tokens: 1.165303E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.361282E-02 | lm-token-buffer-mib: 2.328493E+02 | global_load_balancing_loss: 1.001033E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11215.669 | params rms: 0.060729 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:23:30.242371 | finish at 2026-10-17 17:54:43 + [2026-10-12 11:34:55] iteration 346/ 2384 | consumed samples: 11072 | elapsed time per iteration (ms): 222564.8 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.34% | FLOPs estimator: workload-aware | logical tokens/s: 75381.2 | valid tokens/s: 75381.2 | physical tokens/s: 76218.9 | loss tokens/s: 57853.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.196497E-01 | lm-global-supervised-tokens: 1.287620E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.101409E-02 | lm-token-buffer-mib: 2.345565E+02 | global_load_balancing_loss: 9.986954E-01 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11215.622 | params rms: 0.060729 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:59:47.147481 | finish at 2026-10-17 17:34:42 + [2026-10-12 11:38:39] iteration 347/ 2384 | consumed samples: 11104 | elapsed time per iteration (ms): 224076.4 | throughput per GPU (TFLOP/s/GPU): 24.4 | MFU 7.83% | FLOPs estimator: workload-aware | logical tokens/s: 74872.5 | valid tokens/s: 74872.5 | physical tokens/s: 75518.8 | loss tokens/s: 55052.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.606418E-01 | lm-global-supervised-tokens: 1.233589E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.429310E-02 | lm-token-buffer-mib: 2.354793E+02 | global_load_balancing_loss: 1.000781E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11215.569 | params rms: 0.060729 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:47:23.696820 | finish at 2026-10-17 18:26:03 + [2026-10-12 11:42:21] iteration 348/ 2384 | consumed samples: 11136 | elapsed time per iteration (ms): 222097.6 | throughput per GPU (TFLOP/s/GPU): 24.6 | MFU 7.90% | FLOPs estimator: workload-aware | logical tokens/s: 75539.6 | valid tokens/s: 75539.6 | physical tokens/s: 75696.6 | loss tokens/s: 54626.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.182152E-01 | lm-global-supervised-tokens: 1.213242E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.793762E-02 | lm-token-buffer-mib: 2.327006E+02 | global_load_balancing_loss: 1.001630E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11215.512 | params rms: 0.060728 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:36:30.709682 | finish at 2026-10-17 17:18:52 + [2026-10-12 11:46:04] iteration 349/ 2384 | consumed samples: 11168 | elapsed time per iteration (ms): 222374.5 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.65% | FLOPs estimator: workload-aware | logical tokens/s: 75445.7 | valid tokens/s: 75445.7 | physical tokens/s: 76324.7 | loss tokens/s: 55915.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.624783E-01 | lm-global-supervised-tokens: 1.243424E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.314168E-02 | lm-token-buffer-mib: 2.358425E+02 | global_load_balancing_loss: 1.001403E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11215.456 | params rms: 0.060728 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:42:12.126496 | finish at 2026-10-17 17:28:16 + [2026-10-12 11:50:12] iteration 350/ 2384 | consumed samples: 11200 | elapsed time per iteration (ms): 247855.2 | throughput per GPU (TFLOP/s/GPU): 17.6 | MFU 5.63% | FLOPs estimator: workload-aware | logical tokens/s: 67689.5 | valid tokens/s: 67689.5 | physical tokens/s: 69024.3 | loss tokens/s: 48413.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.977352E-01 | lm-global-supervised-tokens: 1.199947E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.757517E-02 | lm-token-buffer-mib: 2.375336E+02 | global_load_balancing_loss: 1.000691E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11215.403 | params rms: 0.060728 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:02:17.520066 | finish at 2026-10-18 07:52:29 + [2026-10-12 11:53:55] iteration 351/ 2384 | consumed samples: 11232 | elapsed time per iteration (ms): 223924.0 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.45% | FLOPs estimator: workload-aware | logical tokens/s: 74923.6 | valid tokens/s: 74923.6 | physical tokens/s: 76050.4 | loss tokens/s: 56114.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.394841E-01 | lm-global-supervised-tokens: 1.256540E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.262165E-02 | lm-token-buffer-mib: 2.353889E+02 | global_load_balancing_loss: 9.991482E-01 | loss scale: 1.0 | grad norm: 0.158 | params norm: 11215.351 | params rms: 0.060727 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:27:17.474116 | finish at 2026-10-17 18:21:13 + [2026-10-12 11:57:36] iteration 352/ 2384 | consumed samples: 11264 | elapsed time per iteration (ms): 220078.3 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.96% | FLOPs estimator: workload-aware | logical tokens/s: 76232.9 | valid tokens/s: 76232.9 | physical tokens/s: 76949.8 | loss tokens/s: 55921.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.156951E-01 | lm-global-supervised-tokens: 1.230707E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.096101E-02 | lm-token-buffer-mib: 2.340786E+02 | global_load_balancing_loss: 9.990356E-01 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11215.297 | params rms: 0.060727 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:13:19.186504 | finish at 2026-10-17 16:10:55 + [2026-10-12 12:01:16] iteration 353/ 2384 | consumed samples: 11296 | elapsed time per iteration (ms): 220575.0 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.24% | FLOPs estimator: workload-aware | logical tokens/s: 76061.2 | valid tokens/s: 76061.2 | physical tokens/s: 76598.6 | loss tokens/s: 57297.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.802509E-01 | lm-global-supervised-tokens: 1.263837E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.853776E-02 | lm-token-buffer-mib: 2.342584E+02 | global_load_balancing_loss: 9.997311E-01 | loss scale: 1.0 | grad norm: 0.255 | params norm: 11215.241 | params rms: 0.060727 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:26:27.745199 | finish at 2026-10-17 16:27:44 + [2026-10-12 12:04:55] iteration 354/ 2384 | consumed samples: 11328 | elapsed time per iteration (ms): 219076.6 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.14% | FLOPs estimator: workload-aware | logical tokens/s: 76581.4 | valid tokens/s: 76581.4 | physical tokens/s: 76954.0 | loss tokens/s: 56384.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.119102E-01 | lm-global-supervised-tokens: 1.235254E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.326328E-02 | lm-token-buffer-mib: 2.324739E+02 | global_load_balancing_loss: 9.983933E-01 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11215.189 | params rms: 0.060727 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:32:05.526707 | finish at 2026-10-17 15:37:01 + [2026-10-12 12:08:36] iteration 355/ 2384 | consumed samples: 11360 | elapsed time per iteration (ms): 220422.5 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.83% | FLOPs estimator: workload-aware | logical tokens/s: 76113.7 | valid tokens/s: 76113.7 | physical tokens/s: 76981.5 | loss tokens/s: 61068.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.180293E-01 | lm-global-supervised-tokens: 1.346083E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.557821E-02 | lm-token-buffer-mib: 2.333697E+02 | global_load_balancing_loss: 1.000232E+00 | loss scale: 1.0 | grad norm: 0.174 | params norm: 11215.136 | params rms: 0.060726 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:13:57.258092 | finish at 2026-10-17 16:22:33 + [2026-10-12 12:12:20] iteration 356/ 2384 | consumed samples: 11392 | elapsed time per iteration (ms): 224123.4 | throughput per GPU (TFLOP/s/GPU): 24.7 | MFU 7.90% | FLOPs estimator: workload-aware | logical tokens/s: 74856.9 | valid tokens/s: 74856.9 | physical tokens/s: 75398.5 | loss tokens/s: 55865.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.108613E-01 | lm-global-supervised-tokens: 1.252086E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.763425E-02 | lm-token-buffer-mib: 2.348187E+02 | global_load_balancing_loss: 1.003325E+00 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11215.079 | params rms: 0.060726 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:15:22.189871 | finish at 2026-10-17 18:27:42 + [2026-10-12 12:16:04] iteration 357/ 2384 | consumed samples: 11424 | elapsed time per iteration (ms): 224732.4 | throughput per GPU (TFLOP/s/GPU): 24.8 | MFU 7.96% | FLOPs estimator: workload-aware | logical tokens/s: 74654.1 | valid tokens/s: 74654.1 | physical tokens/s: 75425.7 | loss tokens/s: 53141.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.325652E-01 | lm-global-supervised-tokens: 1.194251E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.439008E-02 | lm-token-buffer-mib: 2.357594E+02 | global_load_balancing_loss: 1.003130E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11215.022 | params rms: 0.060726 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:32:12.624457 | finish at 2026-10-17 18:48:17 + [2026-10-12 12:19:45] iteration 358/ 2384 | consumed samples: 11456 | elapsed time per iteration (ms): 220253.3 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.35% | FLOPs estimator: workload-aware | logical tokens/s: 76172.1 | valid tokens/s: 76172.1 | physical tokens/s: 76822.1 | loss tokens/s: 57009.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.843102E-01 | lm-global-supervised-tokens: 1.255644E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.213092E-02 | lm-token-buffer-mib: 2.325840E+02 | global_load_balancing_loss: 1.000959E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11214.964 | params rms: 0.060725 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:57:13.103103 | finish at 2026-10-17 16:16:58 + [2026-10-12 12:23:31] iteration 359/ 2384 | consumed samples: 11488 | elapsed time per iteration (ms): 226134.2 | throughput per GPU (TFLOP/s/GPU): 24.9 | MFU 7.97% | FLOPs estimator: workload-aware | logical tokens/s: 74191.3 | valid tokens/s: 74191.3 | physical tokens/s: 74504.7 | loss tokens/s: 56851.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.346433E-01 | lm-global-supervised-tokens: 1.285602E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.622496E-02 | lm-token-buffer-mib: 2.329502E+02 | global_load_balancing_loss: 9.993150E-01 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11214.906 | params rms: 0.060725 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 7:12:01.843547 | finish at 2026-10-17 19:35:33 + [2026-10-12 12:27:35] iteration 360/ 2384 | consumed samples: 11520 | elapsed time per iteration (ms): 244646.7 | throughput per GPU (TFLOP/s/GPU): 20.2 | MFU 6.48% | FLOPs estimator: workload-aware | logical tokens/s: 68577.3 | valid tokens/s: 68577.3 | physical tokens/s: 69458.9 | loss tokens/s: 53327.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.579689E-01 | lm-global-supervised-tokens: 1.304638E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.452916E-02 | lm-token-buffer-mib: 2.352685E+02 | global_load_balancing_loss: 9.991683E-01 | loss scale: 1.0 | grad norm: 0.299 | params norm: 11214.849 | params rms: 0.060725 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:32:44.866562 | finish at 2026-10-18 06:00:20 +times across ranks (ms): + save-checkpoint: + rank 0: 24344.12 + rank 1: 24344.21 + rank 2: 24344.13 + rank 3: 24344.13 + rank 4: 24344.12 + rank 5: 24344.13 + rank 6: 24344.14 + rank 7: 24344.13 + rank 8: 24343.19 + rank 9: 24343.22 + rank 10: 24343.23 + rank 11: 24343.18 + rank 12: 24343.20 + rank 13: 24343.17 + rank 14: 24343.19 + rank 15: 24343.17 + rank 16: 24342.44 + rank 17: 24342.47 + rank 18: 24342.44 + rank 19: 24342.45 + rank 20: 24342.43 + rank 21: 24342.42 + rank 22: 24342.46 + rank 23: 24342.42 + rank 24: 24342.88 + rank 25: 24342.92 + rank 26: 24342.88 + rank 27: 24342.88 + rank 28: 24343.25 + rank 29: 24342.92 + rank 30: 24342.77 + rank 31: 24343.18 + rank 32: 24342.71 + rank 33: 24342.71 + rank 34: 24342.53 + rank 35: 24342.69 + rank 36: 24342.65 + rank 37: 24342.70 + rank 38: 24342.62 + rank 39: 24342.70 + rank 40: 24342.31 + rank 41: 24342.28 + rank 42: 24342.32 + rank 43: 24342.30 + rank 44: 24342.27 + rank 45: 24342.28 + rank 46: 24342.27 + rank 47: 24342.27 + rank 48: 24342.65 + rank 49: 24342.67 + rank 50: 24342.69 + rank 51: 24342.72 + rank 52: 24342.67 + rank 53: 24342.71 + rank 54: 24342.66 + rank 55: 24342.64 + rank 56: 24342.31 + rank 57: 24342.37 + rank 58: 24342.33 + rank 59: 24342.18 + rank 60: 24342.34 + rank 61: 24342.30 + rank 62: 24342.33 + rank 63: 24342.29 + [2026-10-12 12:31:47] iteration 361/ 2384 | consumed samples: 11552 | elapsed time per iteration (ms): 227446.9 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.36% | FLOPs estimator: workload-aware | logical tokens/s: 73763.2 | valid tokens/s: 73763.2 | physical tokens/s: 74408.1 | loss tokens/s: 55485.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.925350E-01 | lm-global-supervised-tokens: 1.262001E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.761645E-02 | lm-token-buffer-mib: 2.335757E+02 | global_load_balancing_loss: 9.999867E-01 | loss scale: 1.0 | grad norm: 0.096 | params norm: 11214.792 | params rms: 0.060724 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 7:48:45.121406 | finish at 2026-10-17 20:20:32 + [2026-10-12 12:35:37] iteration 362/ 2384 | consumed samples: 11584 | elapsed time per iteration (ms): 230078.4 | throughput per GPU (TFLOP/s/GPU): 28.0 | MFU 8.96% | FLOPs estimator: workload-aware | logical tokens/s: 72919.3 | valid tokens/s: 72919.3 | physical tokens/s: 73898.7 | loss tokens/s: 54290.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.282366E-01 | lm-global-supervised-tokens: 1.249097E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.941124E-02 | lm-token-buffer-mib: 2.349002E+02 | global_load_balancing_loss: 1.000973E+00 | loss scale: 1.0 | grad norm: 0.506 | params norm: 11214.733 | params rms: 0.060724 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 9:13:38.534715 | finish at 2026-10-17 21:49:16 + [2026-10-12 12:39:18] iteration 363/ 2384 | consumed samples: 11616 | elapsed time per iteration (ms): 220678.7 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.24% | FLOPs estimator: workload-aware | logical tokens/s: 76025.4 | valid tokens/s: 76025.4 | physical tokens/s: 76586.7 | loss tokens/s: 56309.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.564329E-01 | lm-global-supervised-tokens: 1.242639E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.470163E-02 | lm-token-buffer-mib: 2.328392E+02 | global_load_balancing_loss: 1.000116E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11214.671 | params rms: 0.060724 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:53:11.593855 | finish at 2026-10-17 16:32:30 + [2026-10-12 12:43:05] iteration 364/ 2384 | consumed samples: 11648 | elapsed time per iteration (ms): 227296.5 | throughput per GPU (TFLOP/s/GPU): 25.0 | MFU 8.01% | FLOPs estimator: workload-aware | logical tokens/s: 73812.0 | valid tokens/s: 73812.0 | physical tokens/s: 74558.5 | loss tokens/s: 54666.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.599037E-01 | lm-global-supervised-tokens: 1.242540E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.212679E-02 | lm-token-buffer-mib: 2.344292E+02 | global_load_balancing_loss: 1.000800E+00 | loss scale: 1.0 | grad norm: 0.271 | params norm: 11214.610 | params rms: 0.060723 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 7:32:18.936195 | finish at 2026-10-17 20:15:24 + [2026-10-12 12:46:44] iteration 365/ 2384 | consumed samples: 11680 | elapsed time per iteration (ms): 219130.2 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.97% | FLOPs estimator: workload-aware | logical tokens/s: 76562.6 | valid tokens/s: 76562.6 | physical tokens/s: 77164.1 | loss tokens/s: 57353.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.629952E-01 | lm-global-supervised-tokens: 1.256776E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.442973E-02 | lm-token-buffer-mib: 2.327145E+02 | global_load_balancing_loss: 9.996037E-01 | loss scale: 1.0 | grad norm: 0.181 | params norm: 11214.549 | params rms: 0.060723 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:53:43.868803 | finish at 2026-10-17 15:40:28 + [2026-10-12 12:50:27] iteration 366/ 2384 | consumed samples: 11712 | elapsed time per iteration (ms): 222178.2 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.66% | FLOPs estimator: workload-aware | logical tokens/s: 75512.3 | valid tokens/s: 75512.3 | physical tokens/s: 76320.2 | loss tokens/s: 59453.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.957277E-01 | lm-global-supervised-tokens: 1.320924E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.571933E-02 | lm-token-buffer-mib: 2.350067E+02 | global_load_balancing_loss: 1.000143E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11214.487 | params rms: 0.060723 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:32:35.513312 | finish at 2026-10-17 17:23:02 + [2026-10-12 12:54:10] iteration 367/ 2384 | consumed samples: 11744 | elapsed time per iteration (ms): 223043.7 | throughput per GPU (TFLOP/s/GPU): 25.6 | MFU 8.19% | FLOPs estimator: workload-aware | logical tokens/s: 75219.2 | valid tokens/s: 75219.2 | physical tokens/s: 75572.1 | loss tokens/s: 59289.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.256144E-01 | lm-global-supervised-tokens: 1.322406E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.809042E-02 | lm-token-buffer-mib: 2.330742E+02 | global_load_balancing_loss: 1.000600E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11214.425 | params rms: 0.060722 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:57:59.173636 | finish at 2026-10-17 17:52:09 + [2026-10-12 12:57:54] iteration 368/ 2384 | consumed samples: 11776 | elapsed time per iteration (ms): 224132.2 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.25% | FLOPs estimator: workload-aware | logical tokens/s: 74854.1 | valid tokens/s: 74854.0 | physical tokens/s: 76013.7 | loss tokens/s: 57469.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.825384E-01 | lm-global-supervised-tokens: 1.288068E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.831271E-02 | lm-token-buffer-mib: 2.355363E+02 | global_load_balancing_loss: 1.001001E+00 | loss scale: 1.0 | grad norm: 0.534 | params norm: 11214.364 | params rms: 0.060722 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:30:50.465698 | finish at 2026-10-17 18:28:44 + [2026-10-12 13:01:39] iteration 369/ 2384 | consumed samples: 11808 | elapsed time per iteration (ms): 225347.6 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.43% | FLOPs estimator: workload-aware | logical tokens/s: 74450.2 | valid tokens/s: 74450.2 | physical tokens/s: 75510.9 | loss tokens/s: 58548.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.013859E-01 | lm-global-supervised-tokens: 1.319383E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.030491E-02 | lm-token-buffer-mib: 2.357317E+02 | global_load_balancing_loss: 1.002165E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11214.301 | params rms: 0.060722 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:07:55.362082 | finish at 2026-10-17 19:09:35 + [2026-10-12 13:05:44] iteration 370/ 2384 | consumed samples: 11840 | elapsed time per iteration (ms): 244859.6 | throughput per GPU (TFLOP/s/GPU): 19.9 | MFU 6.38% | FLOPs estimator: workload-aware | logical tokens/s: 68517.6 | valid tokens/s: 68517.6 | physical tokens/s: 69308.1 | loss tokens/s: 53327.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.930768E-01 | lm-global-supervised-tokens: 1.305778E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.294126E-02 | lm-token-buffer-mib: 2.363906E+02 | global_load_balancing_loss: 1.002388E+00 | loss scale: 1.0 | grad norm: 0.300 | params norm: 11214.237 | params rms: 0.060721 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:59:07.185077 | finish at 2026-10-18 06:04:51 + [2026-10-12 13:09:25] iteration 371/ 2384 | consumed samples: 11872 | elapsed time per iteration (ms): 221377.3 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.50% | FLOPs estimator: workload-aware | logical tokens/s: 75785.5 | valid tokens/s: 75785.5 | physical tokens/s: 76429.7 | loss tokens/s: 57126.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.320919E-01 | lm-global-supervised-tokens: 1.264660E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.392703E-02 | lm-token-buffer-mib: 2.341680E+02 | global_load_balancing_loss: 1.001595E+00 | loss scale: 1.0 | grad norm: 0.310 | params norm: 11214.176 | params rms: 0.060721 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:47:12.534225 | finish at 2026-10-17 16:56:38 + [2026-10-12 13:13:08] iteration 372/ 2384 | consumed samples: 11904 | elapsed time per iteration (ms): 222808.4 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.07% | FLOPs estimator: workload-aware | logical tokens/s: 75298.6 | valid tokens/s: 75298.6 | physical tokens/s: 76098.3 | loss tokens/s: 57996.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.387532E-01 | lm-global-supervised-tokens: 1.292209E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.700995E-02 | lm-token-buffer-mib: 2.347440E+02 | global_load_balancing_loss: 9.999941E-01 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11214.117 | params rms: 0.060721 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:31:30.425318 | finish at 2026-10-17 17:44:39 + [2026-10-12 13:16:52] iteration 373/ 2384 | consumed samples: 11936 | elapsed time per iteration (ms): 223795.4 | throughput per GPU (TFLOP/s/GPU): 23.7 | MFU 7.61% | FLOPs estimator: workload-aware | logical tokens/s: 74966.6 | valid tokens/s: 74966.6 | physical tokens/s: 75716.3 | loss tokens/s: 56449.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.966233E-01 | lm-global-supervised-tokens: 1.263315E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.682421E-02 | lm-token-buffer-mib: 2.341541E+02 | global_load_balancing_loss: 1.000800E+00 | loss scale: 1.0 | grad norm: 0.173 | params norm: 11214.060 | params rms: 0.060720 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:00:52.492152 | finish at 2026-10-17 18:17:45 + [2026-10-12 13:20:35] iteration 374/ 2384 | consumed samples: 11968 | elapsed time per iteration (ms): 223129.0 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.40% | FLOPs estimator: workload-aware | logical tokens/s: 75190.4 | valid tokens/s: 75190.4 | physical tokens/s: 76005.5 | loss tokens/s: 54328.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.360067E-01 | lm-global-supervised-tokens: 1.212235E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.516665E-02 | lm-token-buffer-mib: 2.325510E+02 | global_load_balancing_loss: 9.999342E-01 | loss scale: 1.0 | grad norm: 0.273 | params norm: 11213.994 | params rms: 0.060720 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:34:49.270728 | finish at 2026-10-17 17:55:24 + [2026-10-12 13:24:23] iteration 375/ 2384 | consumed samples: 12000 | elapsed time per iteration (ms): 228050.8 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.15% | FLOPs estimator: workload-aware | logical tokens/s: 73567.7 | valid tokens/s: 73567.7 | physical tokens/s: 74465.9 | loss tokens/s: 56328.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.438233E-01 | lm-global-supervised-tokens: 1.284585E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 7.931596E-02 | lm-token-buffer-mib: 2.356559E+02 | global_load_balancing_loss: 1.000857E+00 | loss scale: 1.0 | grad norm: 0.114 | params norm: 11213.931 | params rms: 0.060720 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 7:15:54.082757 | finish at 2026-10-17 20:40:17 + [2026-10-12 13:28:07] iteration 376/ 2384 | consumed samples: 12032 | elapsed time per iteration (ms): 224172.1 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.55% | FLOPs estimator: workload-aware | logical tokens/s: 74840.8 | valid tokens/s: 74840.8 | physical tokens/s: 75503.1 | loss tokens/s: 53899.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.455889E-01 | lm-global-supervised-tokens: 1.208271E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.372001E-02 | lm-token-buffer-mib: 2.349600E+02 | global_load_balancing_loss: 9.997230E-01 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11213.872 | params rms: 0.060719 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:02:17.586510 | finish at 2026-10-17 18:30:25 + [2026-10-12 13:31:48] iteration 377/ 2384 | consumed samples: 12064 | elapsed time per iteration (ms): 220517.0 | throughput per GPU (TFLOP/s/GPU): 24.9 | MFU 7.97% | FLOPs estimator: workload-aware | logical tokens/s: 76081.1 | valid tokens/s: 76081.1 | physical tokens/s: 76568.6 | loss tokens/s: 55055.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.071288E-01 | lm-global-supervised-tokens: 1.214061E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.292663E-02 | lm-token-buffer-mib: 2.330813E+02 | global_load_balancing_loss: 1.000383E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11213.817 | params rms: 0.060719 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:56:17.666770 | finish at 2026-10-17 16:28:06 + [2026-10-12 13:35:29] iteration 378/ 2384 | consumed samples: 12096 | elapsed time per iteration (ms): 221334.0 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.41% | FLOPs estimator: workload-aware | logical tokens/s: 75800.3 | valid tokens/s: 75800.3 | physical tokens/s: 76720.8 | loss tokens/s: 56822.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.653598E-01 | lm-global-supervised-tokens: 1.257678E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.184181E-02 | lm-token-buffer-mib: 2.328899E+02 | global_load_balancing_loss: 1.000773E+00 | loss scale: 1.0 | grad norm: 0.269 | params norm: 11213.760 | params rms: 0.060719 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:19:56.015222 | finish at 2026-10-17 16:55:25 + [2026-10-12 13:39:09] iteration 379/ 2384 | consumed samples: 12128 | elapsed time per iteration (ms): 219614.3 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.84% | FLOPs estimator: workload-aware | logical tokens/s: 76393.9 | valid tokens/s: 76393.9 | physical tokens/s: 77022.0 | loss tokens/s: 51608.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.982088E-01 | lm-global-supervised-tokens: 1.133392E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.325328E-02 | lm-token-buffer-mib: 2.329887E+02 | global_load_balancing_loss: 1.001464E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11213.702 | params rms: 0.060719 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:18:46.735581 | finish at 2026-10-17 15:57:56 + [2026-10-12 13:43:55] iteration 380/ 2384 | consumed samples: 12160 | elapsed time per iteration (ms): 286639.4 | throughput per GPU (TFLOP/s/GPU): 18.0 | MFU 5.76% | FLOPs estimator: workload-aware | logical tokens/s: 58530.6 | valid tokens/s: 58530.6 | physical tokens/s: 59995.2 | loss tokens/s: 46225.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.733801E-01 | lm-global-supervised-tokens: 1.325008E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.508665E-02 | lm-token-buffer-mib: 2.336989E+02 | global_load_balancing_loss: 1.001729E+00 | loss scale: 1.0 | grad norm: 0.199 | params norm: 11213.645 | params rms: 0.060718 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 15:33:45.378633 | finish at 2026-10-19 05:17:41 + [2026-10-12 13:47:41] iteration 381/ 2384 | consumed samples: 12192 | elapsed time per iteration (ms): 225519.5 | throughput per GPU (TFLOP/s/GPU): 24.0 | MFU 7.68% | FLOPs estimator: workload-aware | logical tokens/s: 74393.5 | valid tokens/s: 74393.5 | physical tokens/s: 75190.0 | loss tokens/s: 56328.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.811887E-01 | lm-global-supervised-tokens: 1.270309E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.678250E-02 | lm-token-buffer-mib: 2.324070E+02 | global_load_balancing_loss: 1.001296E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11213.588 | params rms: 0.060718 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:28:35.462069 | finish at 2026-10-17 19:16:16 + [2026-10-12 13:51:23] iteration 382/ 2384 | consumed samples: 12224 | elapsed time per iteration (ms): 222404.8 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.69% | FLOPs estimator: workload-aware | logical tokens/s: 75435.4 | valid tokens/s: 75435.4 | physical tokens/s: 76153.8 | loss tokens/s: 57472.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.406943E-01 | lm-global-supervised-tokens: 1.278225E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.727202E-02 | lm-token-buffer-mib: 2.328143E+02 | global_load_balancing_loss: 9.996349E-01 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11213.532 | params rms: 0.060718 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:40:54.412817 | finish at 2026-10-17 17:32:18 + [2026-10-12 13:55:03] iteration 383/ 2384 | consumed samples: 12256 | elapsed time per iteration (ms): 219903.7 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.81% | FLOPs estimator: workload-aware | logical tokens/s: 76293.3 | valid tokens/s: 76293.3 | physical tokens/s: 77005.9 | loss tokens/s: 59101.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.945024E-01 | lm-global-supervised-tokens: 1.299660E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.428097E-02 | lm-token-buffer-mib: 2.340922E+02 | global_load_balancing_loss: 9.996247E-01 | loss scale: 1.0 | grad norm: 0.124 | params norm: 11213.477 | params rms: 0.060717 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:13:47.280073 | finish at 2026-10-17 16:08:51 + [2026-10-12 13:58:49] iteration 384/ 2384 | consumed samples: 12288 | elapsed time per iteration (ms): 225598.0 | throughput per GPU (TFLOP/s/GPU): 24.1 | MFU 7.71% | FLOPs estimator: workload-aware | logical tokens/s: 74367.7 | valid tokens/s: 74367.7 | physical tokens/s: 75219.1 | loss tokens/s: 57477.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.130793E-01 | lm-global-supervised-tokens: 1.296673E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.575893E-02 | lm-token-buffer-mib: 2.358247E+02 | global_load_balancing_loss: 1.002483E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11213.421 | params rms: 0.060717 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:19:56.098804 | finish at 2026-10-17 19:18:45 + [2026-10-12 14:02:30] iteration 385/ 2384 | consumed samples: 12320 | elapsed time per iteration (ms): 221281.4 | throughput per GPU (TFLOP/s/GPU): 27.1 | MFU 8.68% | FLOPs estimator: workload-aware | logical tokens/s: 75818.3 | valid tokens/s: 75818.3 | physical tokens/s: 75992.9 | loss tokens/s: 55068.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.151815E-01 | lm-global-supervised-tokens: 1.218573E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.328522E-02 | lm-token-buffer-mib: 2.328018E+02 | global_load_balancing_loss: 1.002147E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11213.365 | params rms: 0.060717 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:52:21.495177 | finish at 2026-10-17 16:54:52 + [2026-10-12 14:06:16] iteration 386/ 2384 | consumed samples: 12352 | elapsed time per iteration (ms): 225468.7 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.31% | FLOPs estimator: workload-aware | logical tokens/s: 74410.3 | valid tokens/s: 74410.3 | physical tokens/s: 75010.5 | loss tokens/s: 53122.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.754974E-01 | lm-global-supervised-tokens: 1.197740E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.330977E-02 | lm-token-buffer-mib: 2.325142E+02 | global_load_balancing_loss: 9.996600E-01 | loss scale: 1.0 | grad norm: 0.182 | params norm: 11213.304 | params rms: 0.060716 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:08:06.531060 | finish at 2026-10-17 19:14:22 + [2026-10-12 14:09:55] iteration 387/ 2384 | consumed samples: 12384 | elapsed time per iteration (ms): 219574.2 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.89% | FLOPs estimator: workload-aware | logical tokens/s: 76407.9 | valid tokens/s: 76407.9 | physical tokens/s: 76681.9 | loss tokens/s: 56619.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.153548E-01 | lm-global-supervised-tokens: 1.243215E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.021326E-01 | lm-token-buffer-mib: 2.327575E+02 | global_load_balancing_loss: 9.988811E-01 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11213.245 | params rms: 0.060716 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:48:09.731984 | finish at 2026-10-17 15:58:05 + [2026-10-12 14:13:37] iteration 388/ 2384 | consumed samples: 12416 | elapsed time per iteration (ms): 221366.5 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.05% | FLOPs estimator: workload-aware | logical tokens/s: 75789.3 | valid tokens/s: 75789.3 | physical tokens/s: 76256.8 | loss tokens/s: 57706.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.768140E-01 | lm-global-supervised-tokens: 1.277434E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.195135E-02 | lm-token-buffer-mib: 2.338704E+02 | global_load_balancing_loss: 1.000308E+00 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11213.188 | params rms: 0.060716 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:44:07.507153 | finish at 2026-10-17 16:57:44 + [2026-10-12 14:17:22] iteration 389/ 2384 | consumed samples: 12448 | elapsed time per iteration (ms): 224937.5 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.85% | FLOPs estimator: workload-aware | logical tokens/s: 74586.1 | valid tokens/s: 74586.0 | physical tokens/s: 75266.1 | loss tokens/s: 55960.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.487544E-01 | lm-global-supervised-tokens: 1.258754E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.092300E-02 | lm-token-buffer-mib: 2.349504E+02 | global_load_balancing_loss: 9.997199E-01 | loss scale: 1.0 | grad norm: 0.327 | params norm: 11213.133 | params rms: 0.060715 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:39:10.375761 | finish at 2026-10-17 18:56:32 + [2026-10-12 14:21:27] iteration 390/ 2384 | consumed samples: 12480 | elapsed time per iteration (ms): 245532.7 | throughput per GPU (TFLOP/s/GPU): 18.4 | MFU 5.89% | FLOPs estimator: workload-aware | logical tokens/s: 68329.8 | valid tokens/s: 68329.8 | physical tokens/s: 69269.3 | loss tokens/s: 53050.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.731595E-01 | lm-global-supervised-tokens: 1.302574E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.176073E-02 | lm-token-buffer-mib: 2.355426E+02 | global_load_balancing_loss: 1.001675E+00 | loss scale: 1.0 | grad norm: 0.475 | params norm: 11213.082 | params rms: 0.060715 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:59:52.180628 | finish at 2026-10-18 06:21:19 + [2026-10-12 14:25:10] iteration 391/ 2384 | consumed samples: 12512 | elapsed time per iteration (ms): 222421.9 | throughput per GPU (TFLOP/s/GPU): 19.5 | MFU 6.23% | FLOPs estimator: workload-aware | logical tokens/s: 75429.6 | valid tokens/s: 75429.6 | physical tokens/s: 76515.1 | loss tokens/s: 59699.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.357657E-01 | lm-global-supervised-tokens: 1.327840E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.728347E-02 | lm-token-buffer-mib: 2.351317E+02 | global_load_balancing_loss: 1.000521E+00 | loss scale: 1.0 | grad norm: 0.139 | params norm: 11213.031 | params rms: 0.060715 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:08:06.751259 | finish at 2026-10-17 17:33:16 + [2026-10-12 14:28:50] iteration 392/ 2384 | consumed samples: 12544 | elapsed time per iteration (ms): 220689.6 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.85% | FLOPs estimator: workload-aware | logical tokens/s: 76021.7 | valid tokens/s: 76021.7 | physical tokens/s: 76837.5 | loss tokens/s: 55411.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.662661E-01 | lm-global-supervised-tokens: 1.222867E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.378941E-02 | lm-token-buffer-mib: 2.339968E+02 | global_load_balancing_loss: 1.000957E+00 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11212.982 | params rms: 0.060715 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:06:53.686981 | finish at 2026-10-17 16:35:44 + [2026-10-12 14:32:34] iteration 393/ 2384 | consumed samples: 12576 | elapsed time per iteration (ms): 223622.6 | throughput per GPU (TFLOP/s/GPU): 25.8 | MFU 8.25% | FLOPs estimator: workload-aware | logical tokens/s: 75024.6 | valid tokens/s: 75024.6 | physical tokens/s: 75241.1 | loss tokens/s: 56383.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.347969E-01 | lm-global-supervised-tokens: 1.260853E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.216119E-02 | lm-token-buffer-mib: 2.324520E+02 | global_load_balancing_loss: 1.000492E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11212.934 | params rms: 0.060714 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:40:32.584414 | finish at 2026-10-17 18:13:06 + [2026-10-12 14:36:15] iteration 394/ 2384 | consumed samples: 12608 | elapsed time per iteration (ms): 220771.9 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.79% | FLOPs estimator: workload-aware | logical tokens/s: 75993.4 | valid tokens/s: 75993.4 | physical tokens/s: 76477.0 | loss tokens/s: 56615.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.675877E-01 | lm-global-supervised-tokens: 1.249902E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.544890E-02 | lm-token-buffer-mib: 2.333966E+02 | global_load_balancing_loss: 1.002244E+00 | loss scale: 1.0 | grad norm: 0.117 | params norm: 11212.883 | params rms: 0.060714 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:02:16.072338 | finish at 2026-10-17 16:38:31 + [2026-10-12 14:40:00] iteration 395/ 2384 | consumed samples: 12640 | elapsed time per iteration (ms): 225609.9 | throughput per GPU (TFLOP/s/GPU): 23.7 | MFU 7.60% | FLOPs estimator: workload-aware | logical tokens/s: 74363.7 | valid tokens/s: 74363.7 | physical tokens/s: 75183.1 | loss tokens/s: 55347.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.346245E-01 | lm-global-supervised-tokens: 1.248692E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.550997E-02 | lm-token-buffer-mib: 2.340531E+02 | global_load_balancing_loss: 1.002088E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11212.831 | params rms: 0.060714 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:38:58.189258 | finish at 2026-10-17 19:18:58 + [2026-10-12 14:43:48] iteration 396/ 2384 | consumed samples: 12672 | elapsed time per iteration (ms): 227514.9 | throughput per GPU (TFLOP/s/GPU): 18.0 | MFU 5.78% | FLOPs estimator: workload-aware | logical tokens/s: 73741.1 | valid tokens/s: 73741.1 | physical tokens/s: 74862.4 | loss tokens/s: 55297.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.890446E-01 | lm-global-supervised-tokens: 1.258104E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.682683E-02 | lm-token-buffer-mib: 2.377724E+02 | global_load_balancing_loss: 1.001441E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11212.779 | params rms: 0.060714 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:38:19.693183 | finish at 2026-10-17 20:22:07 + [2026-10-12 14:47:31] iteration 397/ 2384 | consumed samples: 12704 | elapsed time per iteration (ms): 223533.6 | throughput per GPU (TFLOP/s/GPU): 26.6 | MFU 8.53% | FLOPs estimator: workload-aware | logical tokens/s: 75054.5 | valid tokens/s: 75054.5 | physical tokens/s: 75760.3 | loss tokens/s: 60566.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.978392E-01 | lm-global-supervised-tokens: 1.353856E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.217544E-02 | lm-token-buffer-mib: 2.348644E+02 | global_load_balancing_loss: 1.000650E+00 | loss scale: 1.0 | grad norm: 0.148 | params norm: 11212.728 | params rms: 0.060713 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:22:41.170530 | finish at 2026-10-17 18:10:12 + [2026-10-12 14:51:14] iteration 398/ 2384 | consumed samples: 12736 | elapsed time per iteration (ms): 222360.8 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.02% | FLOPs estimator: workload-aware | logical tokens/s: 75450.3 | valid tokens/s: 75450.3 | physical tokens/s: 76136.3 | loss tokens/s: 57011.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.037004E-01 | lm-global-supervised-tokens: 1.267720E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.892513E-02 | lm-token-buffer-mib: 2.333229E+02 | global_load_balancing_loss: 9.996143E-01 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11212.678 | params rms: 0.060713 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:40:08.555484 | finish at 2026-10-17 17:31:22 + [2026-10-12 14:54:57] iteration 399/ 2384 | consumed samples: 12768 | elapsed time per iteration (ms): 223121.4 | throughput per GPU (TFLOP/s/GPU): 23.7 | MFU 7.60% | FLOPs estimator: workload-aware | logical tokens/s: 75193.2 | valid tokens/s: 75193.2 | physical tokens/s: 75952.5 | loss tokens/s: 58620.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.689251E-01 | lm-global-supervised-tokens: 1.307945E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.710530E-02 | lm-token-buffer-mib: 2.351126E+02 | global_load_balancing_loss: 1.000486E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11212.625 | params rms: 0.060713 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:01:35.953678 | finish at 2026-10-17 17:56:33 + [2026-10-12 14:58:59] iteration 400/ 2384 | consumed samples: 12800 | elapsed time per iteration (ms): 241778.6 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.41% | FLOPs estimator: workload-aware | logical tokens/s: 69390.8 | valid tokens/s: 69390.8 | physical tokens/s: 69858.0 | loss tokens/s: 50908.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.680460E-01 | lm-global-supervised-tokens: 1.230868E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.372385E-02 | lm-token-buffer-mib: 2.332155E+02 | global_load_balancing_loss: 9.991992E-01 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11212.573 | params rms: 0.060712 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:14:48.657211 | finish at 2026-10-18 04:13:47 +times across ranks (ms): + save-checkpoint: + rank 0: 25593.10 + rank 1: 25593.10 + rank 2: 25593.12 + rank 3: 25593.11 + rank 4: 25593.11 + rank 5: 25593.10 + rank 6: 25592.99 + rank 7: 25593.12 + rank 8: 25593.03 + rank 9: 25593.05 + rank 10: 25593.06 + rank 11: 25593.04 + rank 12: 25593.04 + rank 13: 25593.07 + rank 14: 25593.05 + rank 15: 25593.07 + rank 16: 25592.91 + rank 17: 25592.92 + rank 18: 25592.93 + rank 19: 25592.92 + rank 20: 25592.90 + rank 21: 25592.89 + rank 22: 25592.91 + rank 23: 25592.90 + rank 24: 25592.81 + rank 25: 25592.68 + rank 26: 25593.47 + rank 27: 25592.78 + rank 28: 25592.73 + rank 29: 25592.79 + rank 30: 25592.75 + rank 31: 25592.73 + rank 32: 25593.32 + rank 33: 25593.30 + rank 34: 25593.31 + rank 35: 25593.34 + rank 36: 25593.25 + rank 37: 25593.31 + rank 38: 25593.26 + rank 39: 25593.24 + rank 40: 25593.27 + rank 41: 25593.26 + rank 42: 25593.26 + rank 43: 25593.27 + rank 44: 25593.24 + rank 45: 25593.26 + rank 46: 25593.24 + rank 47: 25593.26 + rank 48: 25593.49 + rank 49: 25593.46 + rank 50: 25593.41 + rank 51: 25593.45 + rank 52: 25593.46 + rank 53: 25593.39 + rank 54: 25593.39 + rank 55: 25593.44 + rank 56: 25592.73 + rank 57: 25592.56 + rank 58: 25592.76 + rank 59: 25592.75 + rank 60: 25592.77 + rank 61: 25592.71 + rank 62: 25592.71 + rank 63: 25592.75 + [2026-10-12 15:03:08] iteration 401/ 2384 | consumed samples: 12832 | elapsed time per iteration (ms): 224154.1 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.20% | FLOPs estimator: workload-aware | logical tokens/s: 74846.7 | valid tokens/s: 74846.7 | physical tokens/s: 75360.2 | loss tokens/s: 54920.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.456723E-01 | lm-global-supervised-tokens: 1.231067E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 9.157430E-02 | lm-token-buffer-mib: 2.338413E+02 | global_load_balancing_loss: 1.002198E+00 | loss scale: 1.0 | grad norm: 0.151 | params norm: 11212.520 | params rms: 0.060712 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:28:17.580184 | finish at 2026-10-17 18:31:26 + [2026-10-12 15:06:54] iteration 402/ 2384 | consumed samples: 12864 | elapsed time per iteration (ms): 225555.4 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.26% | FLOPs estimator: workload-aware | logical tokens/s: 74381.6 | valid tokens/s: 74381.6 | physical tokens/s: 75035.0 | loss tokens/s: 58210.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.443243E-01 | lm-global-supervised-tokens: 1.312977E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.340086E-02 | lm-token-buffer-mib: 2.327129E+02 | global_load_balancing_loss: 9.993671E-01 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11212.470 | params rms: 0.060712 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:10:50.897100 | finish at 2026-10-17 19:17:45 + [2026-10-12 15:10:35] iteration 403/ 2384 | consumed samples: 12896 | elapsed time per iteration (ms): 221264.9 | throughput per GPU (TFLOP/s/GPU): 24.3 | MFU 7.80% | FLOPs estimator: workload-aware | logical tokens/s: 75824.0 | valid tokens/s: 75824.0 | physical tokens/s: 76243.5 | loss tokens/s: 60552.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.184148E-01 | lm-global-supervised-tokens: 1.339813E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.825737E-02 | lm-token-buffer-mib: 2.335921E+02 | global_load_balancing_loss: 9.993334E-01 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11212.418 | params rms: 0.060712 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:45:25.838629 | finish at 2026-10-17 16:56:01 + [2026-10-12 15:14:19] iteration 404/ 2384 | consumed samples: 12928 | elapsed time per iteration (ms): 223381.8 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.62% | FLOPs estimator: workload-aware | logical tokens/s: 75105.4 | valid tokens/s: 75105.4 | physical tokens/s: 76336.7 | loss tokens/s: 58038.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.484210E-01 | lm-global-supervised-tokens: 1.296479E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.439536E-02 | lm-token-buffer-mib: 2.370970E+02 | global_load_balancing_loss: 9.996281E-01 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11212.371 | params rms: 0.060711 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:51:35.956793 | finish at 2026-10-17 18:05:54 + [2026-10-12 15:18:00] iteration 405/ 2384 | consumed samples: 12960 | elapsed time per iteration (ms): 221662.7 | throughput per GPU (TFLOP/s/GPU): 24.6 | MFU 7.90% | FLOPs estimator: workload-aware | logical tokens/s: 75687.9 | valid tokens/s: 75687.9 | physical tokens/s: 76254.8 | loss tokens/s: 60185.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.472794E-01 | lm-global-supervised-tokens: 1.334085E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.164460E-02 | lm-token-buffer-mib: 2.326915E+02 | global_load_balancing_loss: 1.001909E+00 | loss scale: 1.0 | grad norm: 0.095 | params norm: 11212.326 | params rms: 0.060711 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:51:10.518092 | finish at 2026-10-17 17:09:11 + [2026-10-12 15:21:46] iteration 406/ 2384 | consumed samples: 12992 | elapsed time per iteration (ms): 226120.5 | throughput per GPU (TFLOP/s/GPU): 26.9 | MFU 8.62% | FLOPs estimator: workload-aware | logical tokens/s: 74195.5 | valid tokens/s: 74195.5 | physical tokens/s: 74753.8 | loss tokens/s: 56310.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.360933E-01 | lm-global-supervised-tokens: 1.273304E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.724912E-02 | lm-token-buffer-mib: 2.340874E+02 | global_load_balancing_loss: 1.001128E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11212.281 | params rms: 0.060711 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:14:26.352475 | finish at 2026-10-17 19:36:13 + [2026-10-12 15:25:30] iteration 407/ 2384 | consumed samples: 13024 | elapsed time per iteration (ms): 223951.2 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.26% | FLOPs estimator: workload-aware | logical tokens/s: 74914.6 | valid tokens/s: 74914.6 | physical tokens/s: 75609.6 | loss tokens/s: 56192.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.425485E-01 | lm-global-supervised-tokens: 1.258437E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.345489E-02 | lm-token-buffer-mib: 2.339507E+02 | global_load_balancing_loss: 1.002100E+00 | loss scale: 1.0 | grad norm: 0.151 | params norm: 11212.237 | params rms: 0.060711 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:59:11.424846 | finish at 2026-10-17 18:24:42 + [2026-10-12 15:29:13] iteration 408/ 2384 | consumed samples: 13056 | elapsed time per iteration (ms): 222684.7 | throughput per GPU (TFLOP/s/GPU): 24.6 | MFU 7.88% | FLOPs estimator: workload-aware | logical tokens/s: 75340.6 | valid tokens/s: 75340.6 | physical tokens/s: 76055.2 | loss tokens/s: 54979.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.165957E-01 | lm-global-supervised-tokens: 1.224299E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.328037E-02 | lm-token-buffer-mib: 2.336228E+02 | global_load_balancing_loss: 1.000517E+00 | loss scale: 1.0 | grad norm: 0.223 | params norm: 11212.192 | params rms: 0.060710 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:13:44.954504 | finish at 2026-10-17 17:42:58 + [2026-10-12 15:32:55] iteration 409/ 2384 | consumed samples: 13088 | elapsed time per iteration (ms): 222153.5 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.66% | FLOPs estimator: workload-aware | logical tokens/s: 75520.8 | valid tokens/s: 75520.8 | physical tokens/s: 76636.0 | loss tokens/s: 54207.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.632878E-01 | lm-global-supervised-tokens: 1.204236E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.999228E-02 | lm-token-buffer-mib: 2.343073E+02 | global_load_balancing_loss: 1.000054E+00 | loss scale: 1.0 | grad norm: 0.268 | params norm: 11212.146 | params rms: 0.060710 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:52:33.065187 | finish at 2026-10-17 17:25:28 + [2026-10-12 15:36:59] iteration 410/ 2384 | consumed samples: 13120 | elapsed time per iteration (ms): 243881.9 | throughput per GPU (TFLOP/s/GPU): 23.9 | MFU 7.65% | FLOPs estimator: workload-aware | logical tokens/s: 68792.3 | valid tokens/s: 68792.3 | physical tokens/s: 69294.4 | loss tokens/s: 53567.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 9.176591E-01 | lm-global-supervised-tokens: 1.306402E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.492695E-02 | lm-token-buffer-mib: 2.334133E+02 | global_load_balancing_loss: 9.990575E-01 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11212.101 | params rms: 0.060710 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:43:42.839680 | finish at 2026-10-18 05:20:42 + [2026-10-12 15:40:41] iteration 411/ 2384 | consumed samples: 13152 | elapsed time per iteration (ms): 221988.5 | throughput per GPU (TFLOP/s/GPU): 24.2 | MFU 7.75% | FLOPs estimator: workload-aware | logical tokens/s: 75576.7 | valid tokens/s: 75576.7 | physical tokens/s: 76271.8 | loss tokens/s: 58680.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.863357E-01 | lm-global-supervised-tokens: 1.302637E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.459828E-02 | lm-token-buffer-mib: 2.342469E+02 | global_load_balancing_loss: 9.998680E-01 | loss scale: 1.0 | grad norm: 0.253 | params norm: 11212.056 | params rms: 0.060710 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:39:43.362948 | finish at 2026-10-17 17:20:24 + [2026-10-12 15:44:29] iteration 412/ 2384 | consumed samples: 13184 | elapsed time per iteration (ms): 227720.2 | throughput per GPU (TFLOP/s/GPU): 25.8 | MFU 8.27% | FLOPs estimator: workload-aware | logical tokens/s: 73674.6 | valid tokens/s: 73674.6 | physical tokens/s: 73981.3 | loss tokens/s: 55598.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.931722E-01 | lm-global-supervised-tokens: 1.266087E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.241493E-02 | lm-token-buffer-mib: 2.329836E+02 | global_load_balancing_loss: 1.002981E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11212.007 | params rms: 0.060709 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:44:24.197849 | finish at 2026-10-17 20:28:53 + [2026-10-12 15:48:15] iteration 413/ 2384 | consumed samples: 13216 | elapsed time per iteration (ms): 226079.4 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.45% | FLOPs estimator: workload-aware | logical tokens/s: 74209.3 | valid tokens/s: 74209.3 | physical tokens/s: 75325.6 | loss tokens/s: 51729.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.611777E-01 | lm-global-supervised-tokens: 1.169504E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.355954E-02 | lm-token-buffer-mib: 2.358356E+02 | global_load_balancing_loss: 1.001764E+00 | loss scale: 1.0 | grad norm: 0.397 | params norm: 11211.953 | params rms: 0.060709 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:46:42.495174 | finish at 2026-10-17 19:34:57 + [2026-10-12 15:52:01] iteration 414/ 2384 | consumed samples: 13248 | elapsed time per iteration (ms): 226078.6 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.41% | FLOPs estimator: workload-aware | logical tokens/s: 74209.5 | valid tokens/s: 74209.5 | physical tokens/s: 75079.3 | loss tokens/s: 58002.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.324914E-01 | lm-global-supervised-tokens: 1.311311E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.651768E-02 | lm-token-buffer-mib: 2.339889E+02 | global_load_balancing_loss: 9.998171E-01 | loss scale: 1.0 | grad norm: 0.204 | params norm: 11211.900 | params rms: 0.060709 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:42:54.831059 | finish at 2026-10-17 19:34:56 + [2026-10-12 15:55:44] iteration 415/ 2384 | consumed samples: 13280 | elapsed time per iteration (ms): 223604.1 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.05% | FLOPs estimator: workload-aware | logical tokens/s: 75030.8 | valid tokens/s: 75030.8 | physical tokens/s: 75549.8 | loss tokens/s: 62400.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.624650E-01 | lm-global-supervised-tokens: 1.395308E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.246196E-02 | lm-token-buffer-mib: 2.331272E+02 | global_load_balancing_loss: 9.989058E-01 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11211.854 | params rms: 0.060709 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:17:56.425464 | finish at 2026-10-17 18:13:41 + [2026-10-12 15:59:31] iteration 416/ 2384 | consumed samples: 13312 | elapsed time per iteration (ms): 226498.9 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.50% | FLOPs estimator: workload-aware | logical tokens/s: 74071.9 | valid tokens/s: 74071.9 | physical tokens/s: 74509.3 | loss tokens/s: 53251.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.078928E-01 | lm-global-supervised-tokens: 1.206142E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.409408E-02 | lm-token-buffer-mib: 2.337095E+02 | global_load_balancing_loss: 9.990239E-01 | loss scale: 1.0 | grad norm: 0.206 | params norm: 11211.808 | params rms: 0.060708 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:49:09.922817 | finish at 2026-10-17 19:48:41 + [2026-10-12 16:03:12] iteration 417/ 2384 | consumed samples: 13344 | elapsed time per iteration (ms): 221102.6 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.48% | FLOPs estimator: workload-aware | logical tokens/s: 75879.7 | valid tokens/s: 75879.7 | physical tokens/s: 76394.4 | loss tokens/s: 56239.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.852913E-01 | lm-global-supervised-tokens: 1.243463E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.246354E-02 | lm-token-buffer-mib: 2.328845E+02 | global_load_balancing_loss: 9.999898E-01 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11211.757 | params rms: 0.060708 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:48:28.819082 | finish at 2026-10-17 16:51:41 + [2026-10-12 16:06:55] iteration 418/ 2384 | consumed samples: 13376 | elapsed time per iteration (ms): 222816.7 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.37% | FLOPs estimator: workload-aware | logical tokens/s: 75295.8 | valid tokens/s: 75295.8 | physical tokens/s: 76141.6 | loss tokens/s: 59534.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.318370E-01 | lm-global-supervised-tokens: 1.326534E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.787097E-02 | lm-token-buffer-mib: 2.337239E+02 | global_load_balancing_loss: 1.001277E+00 | loss scale: 1.0 | grad norm: 0.198 | params norm: 11211.707 | params rms: 0.060708 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:40:57.708567 | finish at 2026-10-17 17:47:53 + [2026-10-12 16:10:38] iteration 419/ 2384 | consumed samples: 13408 | elapsed time per iteration (ms): 223227.5 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.94% | FLOPs estimator: workload-aware | logical tokens/s: 75157.4 | valid tokens/s: 75157.4 | physical tokens/s: 76091.3 | loss tokens/s: 57599.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.072636E-01 | lm-global-supervised-tokens: 1.285773E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.765816E-02 | lm-token-buffer-mib: 2.360801E+02 | global_load_balancing_loss: 1.002158E+00 | loss scale: 1.0 | grad norm: 0.226 | params norm: 11211.662 | params rms: 0.060708 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:50:41.949817 | finish at 2026-10-17 18:01:20 + [2026-10-12 16:14:48] iteration 420/ 2384 | consumed samples: 13440 | elapsed time per iteration (ms): 249902.8 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.67% | FLOPs estimator: workload-aware | logical tokens/s: 67134.9 | valid tokens/s: 67134.9 | physical tokens/s: 68145.3 | loss tokens/s: 53420.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.386282E-01 | lm-global-supervised-tokens: 1.334988E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.319440E-02 | lm-token-buffer-mib: 2.358864E+02 | global_load_balancing_loss: 1.001312E+00 | loss scale: 1.0 | grad norm: 0.173 | params norm: 11211.619 | params rms: 0.060707 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:20:09.182713 | finish at 2026-10-18 08:34:57 + [2026-10-12 16:18:31] iteration 421/ 2384 | consumed samples: 13472 | elapsed time per iteration (ms): 222626.6 | throughput per GPU (TFLOP/s/GPU): 23.5 | MFU 7.54% | FLOPs estimator: workload-aware | logical tokens/s: 75360.2 | valid tokens/s: 75360.2 | physical tokens/s: 76254.7 | loss tokens/s: 56554.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.295505E-01 | lm-global-supervised-tokens: 1.259062E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.811438E-02 | lm-token-buffer-mib: 2.344466E+02 | global_load_balancing_loss: 1.002131E+00 | loss scale: 1.0 | grad norm: 0.402 | params norm: 11211.576 | params rms: 0.060707 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:23:35.946587 | finish at 2026-10-17 17:42:07 + [2026-10-12 16:22:11] iteration 422/ 2384 | consumed samples: 13504 | elapsed time per iteration (ms): 220258.9 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.42% | FLOPs estimator: workload-aware | logical tokens/s: 76170.4 | valid tokens/s: 76170.4 | physical tokens/s: 77141.8 | loss tokens/s: 58712.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.154279E-01 | lm-global-supervised-tokens: 1.293197E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 5.343947E-02 | lm-token-buffer-mib: 2.353303E+02 | global_load_balancing_loss: 1.000497E+00 | loss scale: 1.0 | grad norm: 0.222 | params norm: 11211.537 | params rms: 0.060707 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:02:27.911605 | finish at 2026-10-17 16:24:39 + [2026-10-12 16:25:53] iteration 423/ 2384 | consumed samples: 13536 | elapsed time per iteration (ms): 222209.0 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.06% | FLOPs estimator: workload-aware | logical tokens/s: 75501.9 | valid tokens/s: 75501.9 | physical tokens/s: 76071.9 | loss tokens/s: 60563.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.639549E-01 | lm-global-supervised-tokens: 1.345774E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.527655E-02 | lm-token-buffer-mib: 2.336232E+02 | global_load_balancing_loss: 1.001105E+00 | loss scale: 1.0 | grad norm: 0.195 | params norm: 11211.495 | params rms: 0.060707 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:02:31.877737 | finish at 2026-10-17 17:28:25 + [2026-10-12 16:29:39] iteration 424/ 2384 | consumed samples: 13568 | elapsed time per iteration (ms): 226318.6 | throughput per GPU (TFLOP/s/GPU): 28.8 | MFU 9.24% | FLOPs estimator: workload-aware | logical tokens/s: 74130.7 | valid tokens/s: 74130.7 | physical tokens/s: 74530.8 | loss tokens/s: 58007.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.843873E-01 | lm-global-supervised-tokens: 1.312821E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.568324E-02 | lm-token-buffer-mib: 2.338009E+02 | global_load_balancing_loss: 1.001598E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11211.450 | params rms: 0.060706 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:13:04.489527 | finish at 2026-10-17 19:42:44 + [2026-10-12 16:33:21] iteration 425/ 2384 | consumed samples: 13600 | elapsed time per iteration (ms): 221542.2 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.97% | FLOPs estimator: workload-aware | logical tokens/s: 75729.1 | valid tokens/s: 75729.1 | physical tokens/s: 76217.4 | loss tokens/s: 56792.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.326539E-01 | lm-global-supervised-tokens: 1.258186E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.694295E-02 | lm-token-buffer-mib: 2.344137E+02 | global_load_balancing_loss: 9.996480E-01 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11211.406 | params rms: 0.060706 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:33:21.197530 | finish at 2026-10-17 17:06:42 + [2026-10-12 16:37:09] iteration 426/ 2384 | consumed samples: 13632 | elapsed time per iteration (ms): 227975.0 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.35% | FLOPs estimator: workload-aware | logical tokens/s: 73592.3 | valid tokens/s: 73592.3 | physical tokens/s: 74738.3 | loss tokens/s: 59278.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.604973E-01 | lm-global-supervised-tokens: 1.351411E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.374462E-02 | lm-token-buffer-mib: 2.326497E+02 | global_load_balancing_loss: 9.988051E-01 | loss scale: 1.0 | grad norm: 0.095 | params norm: 11211.359 | params rms: 0.060706 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:59:34.974655 | finish at 2026-10-17 20:36:44 + [2026-10-12 16:40:54] iteration 427/ 2384 | consumed samples: 13664 | elapsed time per iteration (ms): 224886.0 | throughput per GPU (TFLOP/s/GPU): 28.6 | MFU 9.16% | FLOPs estimator: workload-aware | logical tokens/s: 74603.0 | valid tokens/s: 74603.0 | physical tokens/s: 74797.6 | loss tokens/s: 57822.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.363905E-01 | lm-global-supervised-tokens: 1.300345E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.688281E-02 | lm-token-buffer-mib: 2.324808E+02 | global_load_balancing_loss: 9.992049E-01 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11211.315 | params rms: 0.060706 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:15:01.944765 | finish at 2026-10-17 18:55:56 + [2026-10-12 16:44:41] iteration 428/ 2384 | consumed samples: 13696 | elapsed time per iteration (ms): 227393.8 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.67% | FLOPs estimator: workload-aware | logical tokens/s: 73780.3 | valid tokens/s: 73780.3 | physical tokens/s: 74392.3 | loss tokens/s: 51530.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.929936E-01 | lm-global-supervised-tokens: 1.171768E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.871765E-02 | lm-token-buffer-mib: 2.331430E+02 | global_load_balancing_loss: 1.001840E+00 | loss scale: 1.0 | grad norm: 0.134 | params norm: 11211.271 | params rms: 0.060705 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:33:02.346056 | finish at 2026-10-17 20:17:44 + [2026-10-12 16:48:30] iteration 429/ 2384 | consumed samples: 13728 | elapsed time per iteration (ms): 228626.9 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.81% | FLOPs estimator: workload-aware | logical tokens/s: 73382.3 | valid tokens/s: 73382.3 | physical tokens/s: 74423.6 | loss tokens/s: 53420.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.403390E-01 | lm-global-supervised-tokens: 1.221330E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.454162E-02 | lm-token-buffer-mib: 2.380824E+02 | global_load_balancing_loss: 1.000167E+00 | loss scale: 1.0 | grad norm: 0.130 | params norm: 11211.225 | params rms: 0.060705 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:09:25.661664 | finish at 2026-10-17 20:57:56 + [2026-10-12 16:52:34] iteration 430/ 2384 | consumed samples: 13760 | elapsed time per iteration (ms): 243929.1 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.81% | FLOPs estimator: workload-aware | logical tokens/s: 68778.8 | valid tokens/s: 68778.8 | physical tokens/s: 69214.9 | loss tokens/s: 51752.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.268270E-01 | lm-global-supervised-tokens: 1.262399E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.392841E-02 | lm-token-buffer-mib: 2.337599E+02 | global_load_balancing_loss: 1.002479E+00 | loss scale: 1.0 | grad norm: 0.127 | params norm: 11211.177 | params rms: 0.060705 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:23:57.398113 | finish at 2026-10-18 05:16:31 + [2026-10-12 16:56:17] iteration 431/ 2384 | consumed samples: 13792 | elapsed time per iteration (ms): 223588.6 | throughput per GPU (TFLOP/s/GPU): 24.2 | MFU 7.76% | FLOPs estimator: workload-aware | logical tokens/s: 75036.0 | valid tokens/s: 75036.0 | physical tokens/s: 75412.2 | loss tokens/s: 55748.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.835559E-01 | lm-global-supervised-tokens: 1.246482E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.859441E-02 | lm-token-buffer-mib: 2.331151E+02 | global_load_balancing_loss: 1.001023E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11211.127 | params rms: 0.060705 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:17:48.537041 | finish at 2026-10-17 18:14:06 + [2026-10-12 17:00:00] iteration 432/ 2384 | consumed samples: 13824 | elapsed time per iteration (ms): 222569.1 | throughput per GPU (TFLOP/s/GPU): 24.1 | MFU 7.71% | FLOPs estimator: workload-aware | logical tokens/s: 75379.8 | valid tokens/s: 75379.8 | physical tokens/s: 76109.9 | loss tokens/s: 56529.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.737849E-01 | lm-global-supervised-tokens: 1.258178E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.462971E-02 | lm-token-buffer-mib: 2.345482E+02 | global_load_balancing_loss: 9.993416E-01 | loss scale: 1.0 | grad norm: 0.193 | params norm: 11211.078 | params rms: 0.060704 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:40:54.808548 | finish at 2026-10-17 17:40:55 + [2026-10-12 17:03:42] iteration 433/ 2384 | consumed samples: 13856 | elapsed time per iteration (ms): 221885.7 | throughput per GPU (TFLOP/s/GPU): 25.7 | MFU 8.22% | FLOPs estimator: workload-aware | logical tokens/s: 75611.7 | valid tokens/s: 75611.7 | physical tokens/s: 76694.5 | loss tokens/s: 60126.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.690501E-01 | lm-global-supervised-tokens: 1.334128E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 6.154776E-02 | lm-token-buffer-mib: 2.340989E+02 | global_load_balancing_loss: 1.001116E+00 | loss scale: 1.0 | grad norm: 0.106 | params norm: 11211.022 | params rms: 0.060704 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:14:59.029980 | finish at 2026-10-17 17:18:41 + [2026-10-12 17:07:21] iteration 434/ 2384 | consumed samples: 13888 | elapsed time per iteration (ms): 219623.1 | throughput per GPU (TFLOP/s/GPU): 23.4 | MFU 7.49% | FLOPs estimator: workload-aware | logical tokens/s: 76390.8 | valid tokens/s: 76390.8 | physical tokens/s: 76788.4 | loss tokens/s: 57250.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.045730E-01 | lm-global-supervised-tokens: 1.257349E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.657829E-02 | lm-token-buffer-mib: 2.340616E+02 | global_load_balancing_loss: 1.000298E+00 | loss scale: 1.0 | grad norm: 0.262 | params norm: 11210.969 | params rms: 0.060704 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:57:44.981389 | finish at 2026-10-17 16:05:06 + [2026-10-12 17:11:04] iteration 435/ 2384 | consumed samples: 13920 | elapsed time per iteration (ms): 222994.9 | throughput per GPU (TFLOP/s/GPU): 26.4 | MFU 8.46% | FLOPs estimator: workload-aware | logical tokens/s: 75235.6 | valid tokens/s: 75235.6 | physical tokens/s: 75672.4 | loss tokens/s: 54436.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.784249E-01 | lm-global-supervised-tokens: 1.213902E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.696025E-02 | lm-token-buffer-mib: 2.330172E+02 | global_load_balancing_loss: 1.001359E+00 | loss scale: 1.0 | grad norm: 0.096 | params norm: 11210.908 | params rms: 0.060703 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:43:37.133961 | finish at 2026-10-17 17:54:42 + [2026-10-12 17:14:46] iteration 436/ 2384 | consumed samples: 13952 | elapsed time per iteration (ms): 221371.3 | throughput per GPU (TFLOP/s/GPU): 26.5 | MFU 8.50% | FLOPs estimator: workload-aware | logical tokens/s: 75787.4 | valid tokens/s: 75787.4 | physical tokens/s: 76272.8 | loss tokens/s: 59309.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.722846E-01 | lm-global-supervised-tokens: 1.312951E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.780293E-02 | lm-token-buffer-mib: 2.329076E+02 | global_load_balancing_loss: 9.999103E-01 | loss scale: 1.0 | grad norm: 0.130 | params norm: 11210.859 | params rms: 0.060703 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:47:11.327662 | finish at 2026-10-17 17:01:57 + [2026-10-12 17:18:30] iteration 437/ 2384 | consumed samples: 13984 | elapsed time per iteration (ms): 224512.5 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.76% | FLOPs estimator: workload-aware | logical tokens/s: 74727.2 | valid tokens/s: 74727.2 | physical tokens/s: 75886.1 | loss tokens/s: 59940.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.857175E-01 | lm-global-supervised-tokens: 1.345736E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.880407E-02 | lm-token-buffer-mib: 2.356008E+02 | global_load_balancing_loss: 9.991078E-01 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11210.817 | params rms: 0.060703 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:25:25.849198 | finish at 2026-10-17 18:43:56 + [2026-10-12 17:22:18] iteration 438/ 2384 | consumed samples: 14016 | elapsed time per iteration (ms): 228022.1 | throughput per GPU (TFLOP/s/GPU): 19.9 | MFU 6.38% | FLOPs estimator: workload-aware | logical tokens/s: 73081.6 | valid tokens/s: 73081.6 | physical tokens/s: 74857.3 | loss tokens/s: 57079.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.701737E-01 | lm-global-supervised-tokens: 1.301539E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.532914E-02 | lm-token-buffer-mib: 2.375914E+02 | global_load_balancing_loss: 1.000509E+00 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11210.782 | params rms: 0.060703 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:15:31.038095 | finish at 2026-10-17 20:37:49 + [2026-10-12 17:25:59] iteration 439/ 2384 | consumed samples: 14048 | elapsed time per iteration (ms): 220762.4 | throughput per GPU (TFLOP/s/GPU): 25.0 | MFU 8.00% | FLOPs estimator: workload-aware | logical tokens/s: 75996.4 | valid tokens/s: 75996.4 | physical tokens/s: 76260.2 | loss tokens/s: 57882.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.521114E-01 | lm-global-supervised-tokens: 1.277819E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.515083E-02 | lm-token-buffer-mib: 2.333677E+02 | global_load_balancing_loss: 1.002056E+00 | loss scale: 1.0 | grad norm: 0.144 | params norm: 11210.751 | params rms: 0.060703 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:16:22.959182 | finish at 2026-10-17 16:42:22 + [2026-10-12 17:30:04] iteration 440/ 2384 | consumed samples: 14080 | elapsed time per iteration (ms): 245217.5 | throughput per GPU (TFLOP/s/GPU): 19.6 | MFU 6.29% | FLOPs estimator: workload-aware | logical tokens/s: 68417.4 | valid tokens/s: 68417.4 | physical tokens/s: 69201.2 | loss tokens/s: 47390.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.733702E-01 | lm-global-supervised-tokens: 1.162097E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.931553E-02 | lm-token-buffer-mib: 2.335258E+02 | global_load_balancing_loss: 9.999371E-01 | loss scale: 1.0 | grad norm: 0.123 | params norm: 11210.722 | params rms: 0.060702 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:25:02.730492 | finish at 2026-10-18 05:55:07 +times across ranks (ms): + save-checkpoint: + rank 0: 24712.71 + rank 1: 24712.73 + rank 2: 24712.74 + rank 3: 24712.71 + rank 4: 24712.73 + rank 5: 24712.72 + rank 6: 24712.73 + rank 7: 24712.71 + rank 8: 24712.57 + rank 9: 24712.67 + rank 10: 24712.83 + rank 11: 24712.56 + rank 12: 24712.80 + rank 13: 24712.74 + rank 14: 24712.56 + rank 15: 24712.57 + rank 16: 24712.39 + rank 17: 24712.40 + rank 18: 24712.40 + rank 19: 24712.40 + rank 20: 24712.36 + rank 21: 24712.39 + rank 22: 24712.39 + rank 23: 24712.39 + rank 24: 24712.78 + rank 25: 24712.79 + rank 26: 24712.78 + rank 27: 24712.77 + rank 28: 24712.75 + rank 29: 24712.82 + rank 30: 24712.80 + rank 31: 24712.81 + rank 32: 24713.06 + rank 33: 24712.97 + rank 34: 24712.88 + rank 35: 24713.01 + rank 36: 24712.93 + rank 37: 24713.00 + rank 38: 24712.90 + rank 39: 24712.73 + rank 40: 24713.04 + rank 41: 24713.02 + rank 42: 24712.94 + rank 43: 24713.03 + rank 44: 24712.98 + rank 45: 24713.00 + rank 46: 24713.00 + rank 47: 24713.03 + rank 48: 24713.22 + rank 49: 24713.13 + rank 50: 24713.29 + rank 51: 24713.24 + rank 52: 24713.07 + rank 53: 24713.18 + rank 54: 24713.09 + rank 55: 24713.07 + rank 56: 24712.71 + rank 57: 24712.73 + rank 58: 24712.74 + rank 59: 24712.73 + rank 60: 24712.74 + rank 61: 24712.71 + rank 62: 24712.71 + rank 63: 24712.69 + [2026-10-12 17:34:11] iteration 441/ 2384 | consumed samples: 14112 | elapsed time per iteration (ms): 222095.0 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.85% | FLOPs estimator: workload-aware | logical tokens/s: 75540.6 | valid tokens/s: 75540.6 | physical tokens/s: 76244.7 | loss tokens/s: 55592.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.775919E-01 | lm-global-supervised-tokens: 1.234676E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.654020E-02 | lm-token-buffer-mib: 2.342375E+02 | global_load_balancing_loss: 9.999894E-01 | loss scale: 1.0 | grad norm: 0.312 | params norm: 11210.696 | params rms: 0.060702 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:52:10.617483 | finish at 2026-10-17 17:26:22 + [2026-10-12 17:38:02] iteration 442/ 2384 | consumed samples: 14144 | elapsed time per iteration (ms): 231098.0 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.90% | FLOPs estimator: workload-aware | logical tokens/s: 72597.8 | valid tokens/s: 72597.8 | physical tokens/s: 73733.8 | loss tokens/s: 56079.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.255384E-01 | lm-global-supervised-tokens: 1.295975E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.718400E-02 | lm-token-buffer-mib: 2.376130E+02 | global_load_balancing_loss: 1.001446E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11210.671 | params rms: 0.060702 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:39:52.368416 | finish at 2026-10-17 22:17:55 + [2026-10-12 17:41:44] iteration 443/ 2384 | consumed samples: 14176 | elapsed time per iteration (ms): 221591.7 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.55% | FLOPs estimator: workload-aware | logical tokens/s: 75712.2 | valid tokens/s: 75712.2 | physical tokens/s: 76444.5 | loss tokens/s: 56032.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.216341E-01 | lm-global-supervised-tokens: 1.241633E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.027487E-02 | lm-token-buffer-mib: 2.345766E+02 | global_load_balancing_loss: 1.000153E+00 | loss scale: 1.0 | grad norm: 0.326 | params norm: 11210.644 | params rms: 0.060702 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:28:29.450051 | finish at 2026-10-17 17:10:13 + [2026-10-12 17:45:27] iteration 444/ 2384 | consumed samples: 14208 | elapsed time per iteration (ms): 222792.3 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.79% | FLOPs estimator: workload-aware | logical tokens/s: 75304.2 | valid tokens/s: 75304.2 | physical tokens/s: 76369.2 | loss tokens/s: 58411.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.577774E-01 | lm-global-supervised-tokens: 1.301360E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.349970E-02 | lm-token-buffer-mib: 2.336609E+02 | global_load_balancing_loss: 1.000210E+00 | loss scale: 1.0 | grad norm: 0.357 | params norm: 11210.619 | params rms: 0.060702 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:03:37.136440 | finish at 2026-10-17 17:49:04 + [2026-10-12 17:49:08] iteration 445/ 2384 | consumed samples: 14240 | elapsed time per iteration (ms): 221664.1 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.15% | FLOPs estimator: workload-aware | logical tokens/s: 75687.5 | valid tokens/s: 75687.5 | physical tokens/s: 76133.9 | loss tokens/s: 58187.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.368866E-01 | lm-global-supervised-tokens: 1.289799E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.638139E-02 | lm-token-buffer-mib: 2.332752E+02 | global_load_balancing_loss: 1.000543E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11210.592 | params rms: 0.060702 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:23:26.736922 | finish at 2026-10-17 17:12:35 + [2026-10-12 17:52:55] iteration 446/ 2384 | consumed samples: 14272 | elapsed time per iteration (ms): 226545.5 | throughput per GPU (TFLOP/s/GPU): 18.7 | MFU 6.00% | FLOPs estimator: workload-aware | logical tokens/s: 74056.7 | valid tokens/s: 74056.7 | physical tokens/s: 75715.0 | loss tokens/s: 57048.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.662874E-01 | lm-global-supervised-tokens: 1.292408E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.041413E-02 | lm-token-buffer-mib: 2.395870E+02 | global_load_balancing_loss: 1.000968E+00 | loss scale: 1.0 | grad norm: 0.308 | params norm: 11210.564 | params rms: 0.060702 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:57:25.119473 | finish at 2026-10-17 19:50:20 + [2026-10-12 17:56:38] iteration 447/ 2384 | consumed samples: 14304 | elapsed time per iteration (ms): 223363.4 | throughput per GPU (TFLOP/s/GPU): 25.0 | MFU 8.02% | FLOPs estimator: workload-aware | logical tokens/s: 75111.7 | valid tokens/s: 75111.7 | physical tokens/s: 75444.4 | loss tokens/s: 56225.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.279598E-01 | lm-global-supervised-tokens: 1.255875E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.471072E-02 | lm-token-buffer-mib: 2.338156E+02 | global_load_balancing_loss: 1.000301E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11210.535 | params rms: 0.060701 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:10:54.902071 | finish at 2026-10-17 18:07:33 + [2026-10-12 18:00:21] iteration 448/ 2384 | consumed samples: 14336 | elapsed time per iteration (ms): 222921.2 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.76% | FLOPs estimator: workload-aware | logical tokens/s: 75260.6 | valid tokens/s: 75260.6 | physical tokens/s: 75960.1 | loss tokens/s: 58086.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.826558E-01 | lm-global-supervised-tokens: 1.294864E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.239443E-02 | lm-token-buffer-mib: 2.342339E+02 | global_load_balancing_loss: 1.001098E+00 | loss scale: 1.0 | grad norm: 0.374 | params norm: 11210.503 | params rms: 0.060701 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:52:55.506046 | finish at 2026-10-17 17:53:17 + [2026-10-12 18:04:04] iteration 449/ 2384 | consumed samples: 14368 | elapsed time per iteration (ms): 222415.2 | throughput per GPU (TFLOP/s/GPU): 27.1 | MFU 8.69% | FLOPs estimator: workload-aware | logical tokens/s: 75431.9 | valid tokens/s: 75431.9 | physical tokens/s: 75548.8 | loss tokens/s: 54696.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.026020E-01 | lm-global-supervised-tokens: 1.216532E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.242711E-02 | lm-token-buffer-mib: 2.324684E+02 | global_load_balancing_loss: 9.997557E-01 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11210.467 | params rms: 0.060701 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:32:53.394459 | finish at 2026-10-17 17:36:57 + [2026-10-12 18:08:08] iteration 450/ 2384 | consumed samples: 14400 | elapsed time per iteration (ms): 244868.7 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.44% | FLOPs estimator: workload-aware | logical tokens/s: 68515.0 | valid tokens/s: 68515.0 | physical tokens/s: 68917.6 | loss tokens/s: 53127.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.555523E-01 | lm-global-supervised-tokens: 1.300922E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.940934E-02 | lm-token-buffer-mib: 2.324163E+02 | global_load_balancing_loss: 1.001091E+00 | loss scale: 1.0 | grad norm: 0.236 | params norm: 11210.424 | params rms: 0.060701 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:32:56.157612 | finish at 2026-10-18 05:41:05 + [2026-10-12 18:11:57] iteration 451/ 2384 | consumed samples: 14432 | elapsed time per iteration (ms): 228727.5 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.58% | FLOPs estimator: workload-aware | logical tokens/s: 73350.1 | valid tokens/s: 73350.1 | physical tokens/s: 74601.8 | loss tokens/s: 50735.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.350280E-01 | lm-global-supervised-tokens: 1.160449E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.941676E-02 | lm-token-buffer-mib: 2.381002E+02 | global_load_balancing_loss: 9.994854E-01 | loss scale: 1.0 | grad norm: 0.780 | params norm: 11210.375 | params rms: 0.060701 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:48:50.162949 | finish at 2026-10-17 21:00:47 + [2026-10-12 18:15:52] iteration 452/ 2384 | consumed samples: 14464 | elapsed time per iteration (ms): 234783.9 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.78% | FLOPs estimator: workload-aware | logical tokens/s: 71458.0 | valid tokens/s: 71458.0 | physical tokens/s: 72073.1 | loss tokens/s: 51546.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.367920E-01 | lm-global-supervised-tokens: 1.210230E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.747962E-02 | lm-token-buffer-mib: 2.347495E+02 | global_load_balancing_loss: 1.000223E+00 | loss scale: 1.0 | grad norm: 0.137 | params norm: 11210.326 | params rms: 0.060700 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:00:02.479182 | finish at 2026-10-18 00:15:54 + [2026-10-12 18:19:38] iteration 453/ 2384 | consumed samples: 14496 | elapsed time per iteration (ms): 226448.9 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.62% | FLOPs estimator: workload-aware | logical tokens/s: 74088.2 | valid tokens/s: 74088.2 | physical tokens/s: 74764.1 | loss tokens/s: 57367.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.119218E-01 | lm-global-supervised-tokens: 1.299071E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.307845E+00 | lm-token-buffer-mib: 2.333520E+02 | global_load_balancing_loss: 1.000584E+00 | loss scale: 1.0 | grad norm: 0.166 | params norm: 11210.281 | params rms: 0.060700 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:27:52.875131 | finish at 2026-10-17 19:47:31 + [2026-10-12 18:23:24] iteration 454/ 2384 | consumed samples: 14528 | elapsed time per iteration (ms): 225915.3 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.68% | FLOPs estimator: workload-aware | logical tokens/s: 74263.2 | valid tokens/s: 74263.2 | physical tokens/s: 75252.0 | loss tokens/s: 57748.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.360667E-01 | lm-global-supervised-tokens: 1.304630E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.408837E-02 | lm-token-buffer-mib: 2.336317E+02 | global_load_balancing_loss: 1.000963E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11210.239 | params rms: 0.060700 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:06:56.499343 | finish at 2026-10-17 19:30:21 + [2026-10-12 18:27:12] iteration 455/ 2384 | consumed samples: 14560 | elapsed time per iteration (ms): 227691.4 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.37% | FLOPs estimator: workload-aware | logical tokens/s: 73684.0 | valid tokens/s: 73684.0 | physical tokens/s: 74502.5 | loss tokens/s: 55599.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.804605E-01 | lm-global-supervised-tokens: 1.265943E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.606216E-02 | lm-token-buffer-mib: 2.351246E+02 | global_load_balancing_loss: 1.000016E+00 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11210.195 | params rms: 0.060700 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:00:16.779225 | finish at 2026-10-17 20:27:29 + [2026-10-12 18:30:57] iteration 456/ 2384 | consumed samples: 14592 | elapsed time per iteration (ms): 224998.0 | throughput per GPU (TFLOP/s/GPU): 24.2 | MFU 7.74% | FLOPs estimator: workload-aware | logical tokens/s: 74566.0 | valid tokens/s: 74566.0 | physical tokens/s: 75461.8 | loss tokens/s: 59435.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.220568E-01 | lm-global-supervised-tokens: 1.337283E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.720002E-02 | lm-token-buffer-mib: 2.355744E+02 | global_load_balancing_loss: 1.001350E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11210.151 | params rms: 0.060699 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:29:56.223803 | finish at 2026-10-17 19:00:53 + [2026-10-12 18:34:39] iteration 457/ 2384 | consumed samples: 14624 | elapsed time per iteration (ms): 222386.7 | throughput per GPU (TFLOP/s/GPU): 26.2 | MFU 8.41% | FLOPs estimator: workload-aware | logical tokens/s: 75441.3 | valid tokens/s: 75441.3 | physical tokens/s: 76175.2 | loss tokens/s: 62854.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.164437E-01 | lm-global-supervised-tokens: 1.397794E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.307570E-02 | lm-token-buffer-mib: 2.358026E+02 | global_load_balancing_loss: 1.000829E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11210.107 | params rms: 0.060699 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:02:19.144089 | finish at 2026-10-17 17:36:59 + [2026-10-12 18:38:30] iteration 458/ 2384 | consumed samples: 14656 | elapsed time per iteration (ms): 230135.2 | throughput per GPU (TFLOP/s/GPU): 26.2 | MFU 8.39% | FLOPs estimator: workload-aware | logical tokens/s: 72901.3 | valid tokens/s: 72901.3 | physical tokens/s: 73438.6 | loss tokens/s: 54210.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.275271E-01 | lm-global-supervised-tokens: 1.247580E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.460136E-02 | lm-token-buffer-mib: 2.334305E+02 | global_load_balancing_loss: 1.001437E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11210.062 | params rms: 0.060699 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:07:20.366316 | finish at 2026-10-17 21:45:50 + [2026-10-12 18:42:09] iteration 459/ 2384 | consumed samples: 14688 | elapsed time per iteration (ms): 219354.6 | throughput per GPU (TFLOP/s/GPU): 24.7 | MFU 7.91% | FLOPs estimator: workload-aware | logical tokens/s: 76484.3 | valid tokens/s: 76484.3 | physical tokens/s: 77130.7 | loss tokens/s: 58125.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.842088E-01 | lm-global-supervised-tokens: 1.275010E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.047560E-02 | lm-token-buffer-mib: 2.324044E+02 | global_load_balancing_loss: 1.001846E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11210.014 | params rms: 0.060699 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 21:17:37.509905 | finish at 2026-10-17 15:59:46 + [2026-10-12 18:46:21] iteration 460/ 2384 | consumed samples: 14720 | elapsed time per iteration (ms): 251876.5 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.02% | FLOPs estimator: workload-aware | logical tokens/s: 66608.8 | valid tokens/s: 66608.8 | physical tokens/s: 66804.0 | loss tokens/s: 51441.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.239037E-01 | lm-global-supervised-tokens: 1.295701E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.462018E-02 | lm-token-buffer-mib: 2.327303E+02 | global_load_balancing_loss: 1.000665E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11209.965 | params rms: 0.060698 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:36:50.444965 | finish at 2026-10-18 09:23:11 + [2026-10-12 18:50:04] iteration 461/ 2384 | consumed samples: 14752 | elapsed time per iteration (ms): 223505.7 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.64% | FLOPs estimator: workload-aware | logical tokens/s: 75063.8 | valid tokens/s: 75063.8 | physical tokens/s: 75583.1 | loss tokens/s: 57239.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.390929E-01 | lm-global-supervised-tokens: 1.279328E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.400359E-02 | lm-token-buffer-mib: 2.331534E+02 | global_load_balancing_loss: 1.000319E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11209.916 | params rms: 0.060698 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:23:21.397586 | finish at 2026-10-17 18:13:26 + [2026-10-12 18:53:49] iteration 462/ 2384 | consumed samples: 14784 | elapsed time per iteration (ms): 224628.5 | throughput per GPU (TFLOP/s/GPU): 19.2 | MFU 6.17% | FLOPs estimator: workload-aware | logical tokens/s: 74688.7 | valid tokens/s: 74688.7 | physical tokens/s: 75854.6 | loss tokens/s: 58522.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.852581E-01 | lm-global-supervised-tokens: 1.314572E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.014391E+00 | lm-token-buffer-mib: 2.366915E+02 | global_load_balancing_loss: 1.000827E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11209.870 | params rms: 0.060698 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:55:35.937104 | finish at 2026-10-17 18:49:25 + [2026-10-12 18:57:30] iteration 463/ 2384 | consumed samples: 14816 | elapsed time per iteration (ms): 221273.6 | throughput per GPU (TFLOP/s/GPU): 23.8 | MFU 7.62% | FLOPs estimator: workload-aware | logical tokens/s: 75820.9 | valid tokens/s: 75820.9 | physical tokens/s: 76411.8 | loss tokens/s: 61884.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 9.269680E-01 | lm-global-supervised-tokens: 1.369339E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.286823E-02 | lm-token-buffer-mib: 2.343332E+02 | global_load_balancing_loss: 1.000073E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11209.822 | params rms: 0.060698 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:04:26.569307 | finish at 2026-10-17 17:01:57 + [2026-10-12 19:01:17] iteration 464/ 2384 | consumed samples: 14848 | elapsed time per iteration (ms): 226993.0 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.94% | FLOPs estimator: workload-aware | logical tokens/s: 73910.6 | valid tokens/s: 73910.6 | physical tokens/s: 74366.1 | loss tokens/s: 55440.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.552245E-01 | lm-global-supervised-tokens: 1.258454E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.314221E-02 | lm-token-buffer-mib: 2.340548E+02 | global_load_balancing_loss: 9.988832E-01 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11209.768 | params rms: 0.060697 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:03:46.602173 | finish at 2026-10-17 20:05:04 + [2026-10-12 19:04:59] iteration 465/ 2384 | consumed samples: 14880 | elapsed time per iteration (ms): 222248.5 | throughput per GPU (TFLOP/s/GPU): 24.5 | MFU 7.85% | FLOPs estimator: workload-aware | logical tokens/s: 75488.3 | valid tokens/s: 75488.3 | physical tokens/s: 76002.0 | loss tokens/s: 58492.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.614949E-01 | lm-global-supervised-tokens: 1.299979E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.589873E-02 | lm-token-buffer-mib: 2.340734E+02 | global_load_balancing_loss: 9.996758E-01 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11209.713 | params rms: 0.060697 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:28:14.906938 | finish at 2026-10-17 17:33:14 + [2026-10-12 19:08:43] iteration 466/ 2384 | consumed samples: 14912 | elapsed time per iteration (ms): 223201.5 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.65% | FLOPs estimator: workload-aware | logical tokens/s: 75166.2 | valid tokens/s: 75166.2 | physical tokens/s: 75743.2 | loss tokens/s: 55974.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.026620E-01 | lm-global-supervised-tokens: 1.249368E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.413862E-02 | lm-token-buffer-mib: 2.326724E+02 | global_load_balancing_loss: 1.000452E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11209.656 | params rms: 0.060697 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:55:00.435270 | finish at 2026-10-17 18:03:43 + [2026-10-12 19:12:25] iteration 467/ 2384 | consumed samples: 14944 | elapsed time per iteration (ms): 222185.2 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.40% | FLOPs estimator: workload-aware | logical tokens/s: 75510.0 | valid tokens/s: 75510.0 | physical tokens/s: 75832.4 | loss tokens/s: 53358.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.126897E-01 | lm-global-supervised-tokens: 1.185535E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.316983E-02 | lm-token-buffer-mib: 2.324176E+02 | global_load_balancing_loss: 1.000381E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11209.603 | params rms: 0.060696 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:18:49.067203 | finish at 2026-10-17 17:31:14 + [2026-10-12 19:16:15] iteration 468/ 2384 | consumed samples: 14976 | elapsed time per iteration (ms): 229740.5 | throughput per GPU (TFLOP/s/GPU): 25.0 | MFU 8.00% | FLOPs estimator: workload-aware | logical tokens/s: 73026.7 | valid tokens/s: 73026.7 | physical tokens/s: 73757.2 | loss tokens/s: 52906.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.113462E-01 | lm-global-supervised-tokens: 1.215485E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.714282E-02 | lm-token-buffer-mib: 2.360168E+02 | global_load_balancing_loss: 1.002094E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11209.552 | params rms: 0.060696 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:16:22.739477 | finish at 2026-10-17 21:32:37 + [2026-10-12 19:20:32] iteration 469/ 2384 | consumed samples: 15008 | elapsed time per iteration (ms): 257106.1 | throughput per GPU (TFLOP/s/GPU): 18.8 | MFU 6.01% | FLOPs estimator: workload-aware | logical tokens/s: 65254.0 | valid tokens/s: 65254.0 | physical tokens/s: 65699.1 | loss tokens/s: 48796.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.612397E-01 | lm-global-supervised-tokens: 1.254582E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.685445E-02 | lm-token-buffer-mib: 2.336669E+02 | global_load_balancing_loss: 1.000544E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11209.503 | params rms: 0.060696 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:45:58.216814 | finish at 2026-10-18 12:06:30 + [2026-10-12 19:24:40] iteration 470/ 2384 | consumed samples: 15040 | elapsed time per iteration (ms): 248840.2 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.50% | FLOPs estimator: workload-aware | logical tokens/s: 67421.6 | valid tokens/s: 67421.6 | physical tokens/s: 68366.6 | loss tokens/s: 51469.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.703831E-01 | lm-global-supervised-tokens: 1.280768E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.371188E-02 | lm-token-buffer-mib: 2.345078E+02 | global_load_balancing_loss: 1.000197E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11209.452 | params rms: 0.060696 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:18:00.129922 | finish at 2026-10-18 07:42:41 + [2026-10-12 19:28:21] iteration 471/ 2384 | consumed samples: 15072 | elapsed time per iteration (ms): 220754.4 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.28% | FLOPs estimator: workload-aware | logical tokens/s: 75999.3 | valid tokens/s: 75999.3 | physical tokens/s: 76328.5 | loss tokens/s: 60140.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.081863E-01 | lm-global-supervised-tokens: 1.327633E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.550766E-02 | lm-token-buffer-mib: 2.326884E+02 | global_load_balancing_loss: 9.993095E-01 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11209.402 | params rms: 0.060695 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 21:18:23.152633 | finish at 2026-10-17 16:46:44 + [2026-10-12 19:32:04] iteration 472/ 2384 | consumed samples: 15104 | elapsed time per iteration (ms): 222465.1 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.17% | FLOPs estimator: workload-aware | logical tokens/s: 75414.9 | valid tokens/s: 75414.9 | physical tokens/s: 75912.5 | loss tokens/s: 57153.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.684906E-01 | lm-global-supervised-tokens: 1.271465E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.663486E-02 | lm-token-buffer-mib: 2.343095E+02 | global_load_balancing_loss: 9.997023E-01 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11209.350 | params rms: 0.060695 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:09:13.264002 | finish at 2026-10-17 17:41:17 + [2026-10-12 19:35:45] iteration 473/ 2384 | consumed samples: 15136 | elapsed time per iteration (ms): 221171.0 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.97% | FLOPs estimator: workload-aware | logical tokens/s: 75856.2 | valid tokens/s: 75856.2 | physical tokens/s: 76248.7 | loss tokens/s: 57000.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.388919E-01 | lm-global-supervised-tokens: 1.260683E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.788663E-02 | lm-token-buffer-mib: 2.335128E+02 | global_load_balancing_loss: 1.002853E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11209.296 | params rms: 0.060695 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 21:24:17.865297 | finish at 2026-10-17 17:00:03 + [2026-10-12 19:39:25] iteration 474/ 2384 | consumed samples: 15168 | elapsed time per iteration (ms): 219643.5 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.87% | FLOPs estimator: workload-aware | logical tokens/s: 76383.8 | valid tokens/s: 76383.8 | physical tokens/s: 76690.4 | loss tokens/s: 59707.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.131408E-01 | lm-global-supervised-tokens: 1.311433E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.202061E-02 | lm-token-buffer-mib: 2.331481E+02 | global_load_balancing_loss: 1.001576E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11209.243 | params rms: 0.060694 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:31:59.122057 | finish at 2026-10-17 16:11:24 + [2026-10-12 19:43:13] iteration 475/ 2384 | consumed samples: 15200 | elapsed time per iteration (ms): 228024.1 | throughput per GPU (TFLOP/s/GPU): 23.2 | MFU 7.43% | FLOPs estimator: workload-aware | logical tokens/s: 73576.4 | valid tokens/s: 73576.4 | physical tokens/s: 74578.2 | loss tokens/s: 57888.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.141183E-01 | lm-global-supervised-tokens: 1.319998E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.613023E-02 | lm-token-buffer-mib: 2.372733E+02 | global_load_balancing_loss: 1.000462E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11209.189 | params rms: 0.060694 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:54:58.053449 | finish at 2026-10-17 20:38:11 + [2026-10-12 19:46:54] iteration 476/ 2384 | consumed samples: 15232 | elapsed time per iteration (ms): 221890.1 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.74% | FLOPs estimator: workload-aware | logical tokens/s: 75610.4 | valid tokens/s: 75610.4 | physical tokens/s: 76263.5 | loss tokens/s: 57446.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.329697E-01 | lm-global-supervised-tokens: 1.274673E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.490391E-02 | lm-token-buffer-mib: 2.339839E+02 | global_load_balancing_loss: 1.001326E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11209.132 | params rms: 0.060694 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 21:36:06.293973 | finish at 2026-10-17 17:23:01 + [2026-10-12 19:50:39] iteration 477/ 2384 | consumed samples: 15264 | elapsed time per iteration (ms): 224673.8 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.87% | FLOPs estimator: workload-aware | logical tokens/s: 74673.5 | valid tokens/s: 74673.5 | physical tokens/s: 75289.5 | loss tokens/s: 54128.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.186643E-01 | lm-global-supervised-tokens: 1.216116E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.487568E-02 | lm-token-buffer-mib: 2.345676E+02 | global_load_balancing_loss: 1.000987E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11209.073 | params rms: 0.060693 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:00:52.895210 | finish at 2026-10-17 18:51:32 + [2026-10-12 19:54:28] iteration 478/ 2384 | consumed samples: 15296 | elapsed time per iteration (ms): 229327.9 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.96% | FLOPs estimator: workload-aware | logical tokens/s: 73158.2 | valid tokens/s: 73158.2 | physical tokens/s: 73541.6 | loss tokens/s: 52605.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.850888E-01 | lm-global-supervised-tokens: 1.206381E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.146975E-02 | lm-token-buffer-mib: 2.333446E+02 | global_load_balancing_loss: 9.999295E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11209.016 | params rms: 0.060693 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:24:58.990450 | finish at 2026-10-17 21:19:27 + [2026-10-12 19:58:10] iteration 479/ 2384 | consumed samples: 15328 | elapsed time per iteration (ms): 221582.4 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.10% | FLOPs estimator: workload-aware | logical tokens/s: 75715.4 | valid tokens/s: 75715.4 | physical tokens/s: 76786.5 | loss tokens/s: 53762.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.702485E-01 | lm-global-supervised-tokens: 1.191272E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.067818E-02 | lm-token-buffer-mib: 2.344024E+02 | global_load_balancing_loss: 9.992324E-01 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11208.963 | params rms: 0.060693 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 21:15:14.403577 | finish at 2026-10-17 17:13:24 + [2026-10-12 20:02:12] iteration 480/ 2384 | consumed samples: 15360 | elapsed time per iteration (ms): 242154.8 | throughput per GPU (TFLOP/s/GPU): 18.7 | MFU 6.01% | FLOPs estimator: workload-aware | logical tokens/s: 69283.0 | valid tokens/s: 69283.0 | physical tokens/s: 70662.6 | loss tokens/s: 51509.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.939369E-01 | lm-global-supervised-tokens: 1.247337E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.235509E-02 | lm-token-buffer-mib: 2.338251E+02 | global_load_balancing_loss: 1.000892E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11208.908 | params rms: 0.060693 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:04:22.658733 | finish at 2026-10-18 04:06:35 +times across ranks (ms): + save-checkpoint: + rank 0: 23696.54 + rank 1: 23696.58 + rank 2: 23696.55 + rank 3: 23696.57 + rank 4: 23696.54 + rank 5: 23696.56 + rank 6: 23696.59 + rank 7: 23696.56 + rank 8: 23696.40 + rank 9: 23696.36 + rank 10: 23696.39 + rank 11: 23696.38 + rank 12: 23696.36 + rank 13: 23696.35 + rank 14: 23696.35 + rank 15: 23696.36 + rank 16: 23696.11 + rank 17: 23696.19 + rank 18: 23696.16 + rank 19: 23696.10 + rank 20: 23696.10 + rank 21: 23696.12 + rank 22: 23696.13 + rank 23: 23696.09 + rank 24: 23696.23 + rank 25: 23696.26 + rank 26: 23696.28 + rank 27: 23696.26 + rank 28: 23696.22 + rank 29: 23696.28 + rank 30: 23696.25 + rank 31: 23696.22 + rank 32: 23694.54 + rank 33: 23694.53 + rank 34: 23694.54 + rank 35: 23694.54 + rank 36: 23694.50 + rank 37: 23694.52 + rank 38: 23694.51 + rank 39: 23694.52 + rank 40: 23696.37 + rank 41: 23696.35 + rank 42: 23696.35 + rank 43: 23696.37 + rank 44: 23696.38 + rank 45: 23696.47 + rank 46: 23696.36 + rank 47: 23696.38 + rank 48: 23695.85 + rank 49: 23695.87 + rank 50: 23695.88 + rank 51: 23695.84 + rank 52: 23695.84 + rank 53: 23695.83 + rank 54: 23695.84 + rank 55: 23695.84 + rank 56: 23695.99 + rank 57: 23696.00 + rank 58: 23696.01 + rank 59: 23696.00 + rank 60: 23696.01 + rank 61: 23696.01 + rank 62: 23696.00 + rank 63: 23695.98 + [2026-10-12 20:06:18] iteration 481/ 2384 | consumed samples: 15392 | elapsed time per iteration (ms): 222166.0 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.74% | FLOPs estimator: workload-aware | logical tokens/s: 75516.5 | valid tokens/s: 75516.5 | physical tokens/s: 76227.8 | loss tokens/s: 58839.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.822979E-01 | lm-global-supervised-tokens: 1.307217E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 2.351256E-01 | lm-token-buffer-mib: 2.337592E+02 | global_load_balancing_loss: 1.000712E+00 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11208.850 | params rms: 0.060692 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 21:26:21.975828 | finish at 2026-10-17 17:32:40 + [2026-10-12 20:10:03] iteration 482/ 2384 | consumed samples: 15424 | elapsed time per iteration (ms): 224610.3 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.52% | FLOPs estimator: workload-aware | logical tokens/s: 74694.7 | valid tokens/s: 74694.7 | physical tokens/s: 75425.1 | loss tokens/s: 56390.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.039103E-01 | lm-global-supervised-tokens: 1.266593E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.006542E-02 | lm-token-buffer-mib: 2.331818E+02 | global_load_balancing_loss: 1.001925E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11208.791 | params rms: 0.060692 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:40:08.781199 | finish at 2026-10-17 18:50:11 + [2026-10-12 20:13:42] iteration 483/ 2384 | consumed samples: 15456 | elapsed time per iteration (ms): 219808.0 | throughput per GPU (TFLOP/s/GPU): 23.2 | MFU 7.43% | FLOPs estimator: workload-aware | logical tokens/s: 76326.7 | valid tokens/s: 76326.7 | physical tokens/s: 76832.7 | loss tokens/s: 55970.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.655440E-01 | lm-global-supervised-tokens: 1.230268E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.792181E-02 | lm-token-buffer-mib: 2.341937E+02 | global_load_balancing_loss: 1.001977E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11208.733 | params rms: 0.060692 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:04:14.948793 | finish at 2026-10-17 16:17:57 + [2026-10-12 20:17:29] iteration 484/ 2384 | consumed samples: 15488 | elapsed time per iteration (ms): 226494.6 | throughput per GPU (TFLOP/s/GPU): 17.7 | MFU 5.67% | FLOPs estimator: workload-aware | logical tokens/s: 74073.3 | valid tokens/s: 74073.3 | physical tokens/s: 75543.0 | loss tokens/s: 54333.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.658140E-01 | lm-global-supervised-tokens: 1.230615E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.278933E-02 | lm-token-buffer-mib: 2.390696E+02 | global_load_balancing_loss: 1.001142E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11208.676 | params rms: 0.060691 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:32:19.682579 | finish at 2026-10-17 19:49:49 + [2026-10-12 20:21:39] iteration 485/ 2384 | consumed samples: 15520 | elapsed time per iteration (ms): 250002.9 | throughput per GPU (TFLOP/s/GPU): 17.7 | MFU 5.66% | FLOPs estimator: workload-aware | logical tokens/s: 67108.1 | valid tokens/s: 67108.1 | physical tokens/s: 67928.1 | loss tokens/s: 53333.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.014173E-01 | lm-global-supervised-tokens: 1.333344E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.385506E-02 | lm-token-buffer-mib: 2.338883E+02 | global_load_balancing_loss: 1.000777E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11208.620 | params rms: 0.060691 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:52:35.443043 | finish at 2026-10-18 08:14:14 + [2026-10-12 20:25:31] iteration 486/ 2384 | consumed samples: 15552 | elapsed time per iteration (ms): 231859.6 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.19% | FLOPs estimator: workload-aware | logical tokens/s: 72359.2 | valid tokens/s: 72359.2 | physical tokens/s: 73020.7 | loss tokens/s: 53013.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.881733E-01 | lm-global-supervised-tokens: 1.229161E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.528977E-02 | lm-token-buffer-mib: 2.346482E+02 | global_load_balancing_loss: 9.999171E-01 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11208.561 | params rms: 0.060691 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:14:29.541743 | finish at 2026-10-17 22:40:00 + [2026-10-12 20:29:16] iteration 487/ 2384 | consumed samples: 15584 | elapsed time per iteration (ms): 224968.0 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.94% | FLOPs estimator: workload-aware | logical tokens/s: 74575.9 | valid tokens/s: 74575.9 | physical tokens/s: 75539.8 | loss tokens/s: 54245.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.455594E-01 | lm-global-supervised-tokens: 1.220355E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.138197E-02 | lm-token-buffer-mib: 2.354379E+02 | global_load_balancing_loss: 1.000177E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11208.500 | params rms: 0.060690 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:32:44.321205 | finish at 2026-10-17 19:02:00 + [2026-10-12 20:32:59] iteration 488/ 2384 | consumed samples: 15616 | elapsed time per iteration (ms): 222801.1 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.40% | FLOPs estimator: workload-aware | logical tokens/s: 75301.3 | valid tokens/s: 75301.3 | physical tokens/s: 76567.8 | loss tokens/s: 58118.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.520270E-01 | lm-global-supervised-tokens: 1.294881E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.742239E-02 | lm-token-buffer-mib: 2.359287E+02 | global_load_balancing_loss: 9.996201E-01 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11208.443 | params rms: 0.060690 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 21:20:30.796125 | finish at 2026-10-17 17:53:29 + [2026-10-12 20:36:42] iteration 489/ 2384 | consumed samples: 15648 | elapsed time per iteration (ms): 222952.6 | throughput per GPU (TFLOP/s/GPU): 23.7 | MFU 7.58% | FLOPs estimator: workload-aware | logical tokens/s: 75249.9 | valid tokens/s: 75249.9 | physical tokens/s: 75819.7 | loss tokens/s: 57159.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.450836E-01 | lm-global-supervised-tokens: 1.274393E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.380940E-02 | lm-token-buffer-mib: 2.347963E+02 | global_load_balancing_loss: 1.000290E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11208.389 | params rms: 0.060690 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 21:21:35.251100 | finish at 2026-10-17 17:58:17 + [2026-10-12 20:40:49] iteration 490/ 2384 | consumed samples: 15680 | elapsed time per iteration (ms): 247512.8 | throughput per GPU (TFLOP/s/GPU): 18.8 | MFU 6.03% | FLOPs estimator: workload-aware | logical tokens/s: 67783.0 | valid tokens/s: 67783.0 | physical tokens/s: 68210.4 | loss tokens/s: 51791.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.254409E-01 | lm-global-supervised-tokens: 1.281906E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.698343E-02 | lm-token-buffer-mib: 2.344760E+02 | global_load_balancing_loss: 9.990963E-01 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11208.336 | params rms: 0.060689 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 10:13:09.312248 | finish at 2026-10-18 06:53:58 + [2026-10-12 20:44:30] iteration 491/ 2384 | consumed samples: 15712 | elapsed time per iteration (ms): 221153.3 | throughput per GPU (TFLOP/s/GPU): 25.2 | MFU 8.07% | FLOPs estimator: workload-aware | logical tokens/s: 75862.4 | valid tokens/s: 75862.4 | physical tokens/s: 76083.2 | loss tokens/s: 53201.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.529648E-01 | lm-global-supervised-tokens: 1.176572E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.219873E-02 | lm-token-buffer-mib: 2.330206E+02 | global_load_balancing_loss: 1.000968E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11208.285 | params rms: 0.060689 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:17:23.215042 | finish at 2026-10-17 17:01:53 + [2026-10-12 20:48:14] iteration 492/ 2384 | consumed samples: 15744 | elapsed time per iteration (ms): 223441.5 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.95% | FLOPs estimator: workload-aware | logical tokens/s: 75085.2 | valid tokens/s: 75085.2 | physical tokens/s: 76205.2 | loss tokens/s: 57130.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.251645E-01 | lm-global-supervised-tokens: 1.276534E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.269176E-02 | lm-token-buffer-mib: 2.345852E+02 | global_load_balancing_loss: 1.000503E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11208.236 | params rms: 0.060689 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 21:25:51.242571 | finish at 2026-10-17 18:14:05 + [2026-10-12 20:52:07] iteration 493/ 2384 | consumed samples: 15776 | elapsed time per iteration (ms): 233625.3 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.99% | FLOPs estimator: workload-aware | logical tokens/s: 71812.4 | valid tokens/s: 71812.4 | physical tokens/s: 72887.5 | loss tokens/s: 49405.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.329565E-01 | lm-global-supervised-tokens: 1.154239E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.872506E-02 | lm-token-buffer-mib: 2.378654E+02 | global_load_balancing_loss: 1.001008E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11208.186 | params rms: 0.060689 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:43:05.357409 | finish at 2026-10-17 23:35:13 + [2026-10-12 20:55:53] iteration 494/ 2384 | consumed samples: 15808 | elapsed time per iteration (ms): 226088.0 | throughput per GPU (TFLOP/s/GPU): 19.1 | MFU 6.12% | FLOPs estimator: workload-aware | logical tokens/s: 74206.5 | valid tokens/s: 74206.5 | physical tokens/s: 75007.1 | loss tokens/s: 57116.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.711665E-01 | lm-global-supervised-tokens: 1.291346E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.576781E-02 | lm-token-buffer-mib: 2.346101E+02 | global_load_balancing_loss: 1.001104E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11208.140 | params rms: 0.060688 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:41:46.353457 | finish at 2026-10-17 19:37:40 + [2026-10-12 20:59:36] iteration 495/ 2384 | consumed samples: 15840 | elapsed time per iteration (ms): 222171.0 | throughput per GPU (TFLOP/s/GPU): 19.2 | MFU 6.15% | FLOPs estimator: workload-aware | logical tokens/s: 75514.7 | valid tokens/s: 75514.7 | physical tokens/s: 76119.8 | loss tokens/s: 58809.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.805067E-01 | lm-global-supervised-tokens: 1.306579E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.002213E-02 | lm-token-buffer-mib: 2.328402E+02 | global_load_balancing_loss: 1.000055E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11208.094 | params rms: 0.060688 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:34:41.022610 | finish at 2026-10-17 17:34:17 + [2026-10-12 21:03:17] iteration 496/ 2384 | consumed samples: 15872 | elapsed time per iteration (ms): 221760.7 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.18% | FLOPs estimator: workload-aware | logical tokens/s: 75654.6 | valid tokens/s: 75654.6 | physical tokens/s: 76126.7 | loss tokens/s: 58164.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.865644E-01 | lm-global-supervised-tokens: 1.289848E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.279540E-02 | lm-token-buffer-mib: 2.338953E+02 | global_load_balancing_loss: 1.000435E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11208.046 | params rms: 0.060688 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:18:04.255142 | finish at 2026-10-17 17:21:22 + [2026-10-12 21:06:58] iteration 497/ 2384 | consumed samples: 15904 | elapsed time per iteration (ms): 220663.4 | throughput per GPU (TFLOP/s/GPU): 23.4 | MFU 7.51% | FLOPs estimator: workload-aware | logical tokens/s: 76030.7 | valid tokens/s: 76030.7 | physical tokens/s: 76595.8 | loss tokens/s: 54447.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.664972E-01 | lm-global-supervised-tokens: 1.201467E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.424856E-02 | lm-token-buffer-mib: 2.328245E+02 | global_load_balancing_loss: 1.001694E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11207.997 | params rms: 0.060688 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:39:51.929255 | finish at 2026-10-17 16:46:50 + [2026-10-12 21:10:43] iteration 498/ 2384 | consumed samples: 15936 | elapsed time per iteration (ms): 224801.7 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.35% | FLOPs estimator: workload-aware | logical tokens/s: 74631.0 | valid tokens/s: 74631.0 | physical tokens/s: 75665.7 | loss tokens/s: 56867.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.336050E-01 | lm-global-supervised-tokens: 1.278383E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.375364E-02 | lm-token-buffer-mib: 2.370899E+02 | global_load_balancing_loss: 1.000233E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11207.949 | params rms: 0.060687 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 21:46:16.089604 | finish at 2026-10-17 18:56:59 + [2026-10-12 21:14:30] iteration 499/ 2384 | consumed samples: 15968 | elapsed time per iteration (ms): 227401.3 | throughput per GPU (TFLOP/s/GPU): 25.1 | MFU 8.05% | FLOPs estimator: workload-aware | logical tokens/s: 73777.8 | valid tokens/s: 73777.8 | physical tokens/s: 74254.5 | loss tokens/s: 55078.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.206359E-01 | lm-global-supervised-tokens: 1.252496E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.253661E-02 | lm-token-buffer-mib: 2.337034E+02 | global_load_balancing_loss: 9.999521E-01 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11207.905 | params rms: 0.060687 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:04:11.455805 | finish at 2026-10-17 20:18:42 + [2026-10-12 21:18:36] iteration 500/ 2384 | consumed samples: 16000 | elapsed time per iteration (ms): 245566.0 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.44% | FLOPs estimator: workload-aware | logical tokens/s: 68320.6 | valid tokens/s: 68320.6 | physical tokens/s: 68991.2 | loss tokens/s: 49675.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.416401E-01 | lm-global-supervised-tokens: 1.219856E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.985525E-02 | lm-token-buffer-mib: 2.354858E+02 | global_load_balancing_loss: 9.999008E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11207.860 | params rms: 0.060687 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:30:46.413594 | finish at 2026-10-18 05:49:22 + [2026-10-12 21:22:16] iteration 501/ 2384 | consumed samples: 16032 | elapsed time per iteration (ms): 220608.2 | throughput per GPU (TFLOP/s/GPU): 25.6 | MFU 8.19% | FLOPs estimator: workload-aware | logical tokens/s: 76049.7 | valid tokens/s: 76049.7 | physical tokens/s: 76574.6 | loss tokens/s: 60529.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.600543E-01 | lm-global-supervised-tokens: 1.335320E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.726959E-02 | lm-token-buffer-mib: 2.341679E+02 | global_load_balancing_loss: 1.000535E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11207.813 | params rms: 0.060687 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:23:25.222780 | finish at 2026-10-17 16:45:42 + [2026-10-12 21:25:57] iteration 502/ 2384 | consumed samples: 16064 | elapsed time per iteration (ms): 221005.9 | throughput per GPU (TFLOP/s/GPU): 19.4 | MFU 6.22% | FLOPs estimator: workload-aware | logical tokens/s: 75912.9 | valid tokens/s: 75912.8 | physical tokens/s: 76749.6 | loss tokens/s: 60172.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.734245E-01 | lm-global-supervised-tokens: 1.329838E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.417998E-02 | lm-token-buffer-mib: 2.350063E+02 | global_load_balancing_loss: 1.000218E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11207.765 | params rms: 0.060686 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:32:13.177660 | finish at 2026-10-17 16:58:11 + [2026-10-12 21:29:39] iteration 503/ 2384 | consumed samples: 16096 | elapsed time per iteration (ms): 222013.5 | throughput per GPU (TFLOP/s/GPU): 26.5 | MFU 8.49% | FLOPs estimator: workload-aware | logical tokens/s: 75568.3 | valid tokens/s: 75568.3 | physical tokens/s: 75841.4 | loss tokens/s: 60173.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.745273E-01 | lm-global-supervised-tokens: 1.335941E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.144154E-02 | lm-token-buffer-mib: 2.325118E+02 | global_load_balancing_loss: 1.000416E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11207.719 | params rms: 0.060686 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:00:07.354437 | finish at 2026-10-17 17:29:47 + [2026-10-12 21:33:26] iteration 504/ 2384 | consumed samples: 16128 | elapsed time per iteration (ms): 226309.4 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.13% | FLOPs estimator: workload-aware | logical tokens/s: 74133.8 | valid tokens/s: 74133.8 | physical tokens/s: 74748.5 | loss tokens/s: 53595.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.664813E-01 | lm-global-supervised-tokens: 1.212912E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.372142E-02 | lm-token-buffer-mib: 2.350351E+02 | global_load_balancing_loss: 1.000516E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11207.675 | params rms: 0.060686 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:11:01.625538 | finish at 2026-10-17 19:44:27 + [2026-10-12 21:37:09] iteration 505/ 2384 | consumed samples: 16160 | elapsed time per iteration (ms): 223218.6 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.91% | FLOPs estimator: workload-aware | logical tokens/s: 75160.4 | valid tokens/s: 75160.4 | physical tokens/s: 75785.2 | loss tokens/s: 62627.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.368588E-01 | lm-global-supervised-tokens: 1.397971E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.651759E-02 | lm-token-buffer-mib: 2.339457E+02 | global_load_balancing_loss: 1.002142E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11207.629 | params rms: 0.060686 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:30:27.816216 | finish at 2026-10-17 18:07:37 + [2026-10-12 21:40:54] iteration 506/ 2384 | consumed samples: 16192 | elapsed time per iteration (ms): 224832.3 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.38% | FLOPs estimator: workload-aware | logical tokens/s: 74620.9 | valid tokens/s: 74620.9 | physical tokens/s: 75284.0 | loss tokens/s: 53833.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.057990E-01 | lm-global-supervised-tokens: 1.210355E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.450466E-02 | lm-token-buffer-mib: 2.343804E+02 | global_load_balancing_loss: 9.998326E-01 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11207.583 | params rms: 0.060685 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 21:17:14.972438 | finish at 2026-10-17 18:58:09 + [2026-10-12 21:44:50] iteration 507/ 2384 | consumed samples: 16224 | elapsed time per iteration (ms): 236316.3 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.43% | FLOPs estimator: workload-aware | logical tokens/s: 70994.7 | valid tokens/s: 70994.7 | physical tokens/s: 71539.9 | loss tokens/s: 56110.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.176676E-01 | lm-global-supervised-tokens: 1.325974E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.808094E-02 | lm-token-buffer-mib: 2.337849E+02 | global_load_balancing_loss: 9.992426E-01 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11207.539 | params rms: 0.060685 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:12:45.626919 | finish at 2026-10-18 00:57:36 + [2026-10-12 21:48:38] iteration 508/ 2384 | consumed samples: 16256 | elapsed time per iteration (ms): 227496.3 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.77% | FLOPs estimator: workload-aware | logical tokens/s: 73747.1 | valid tokens/s: 73747.1 | physical tokens/s: 74477.0 | loss tokens/s: 52161.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.131212E-01 | lm-global-supervised-tokens: 1.186645E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.170354E+00 | lm-token-buffer-mib: 2.347731E+02 | global_load_balancing_loss: 9.990718E-01 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11207.494 | params rms: 0.060685 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:33:02.985414 | finish at 2026-10-17 20:21:41 + [2026-10-12 21:52:20] iteration 509/ 2384 | consumed samples: 16288 | elapsed time per iteration (ms): 222012.5 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.20% | FLOPs estimator: workload-aware | logical tokens/s: 75568.6 | valid tokens/s: 75568.6 | physical tokens/s: 76348.8 | loss tokens/s: 60094.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.128570E-01 | lm-global-supervised-tokens: 1.334174E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.524671E-02 | lm-token-buffer-mib: 2.336554E+02 | global_load_balancing_loss: 1.000981E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11207.446 | params rms: 0.060685 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:37:53.527443 | finish at 2026-10-17 17:30:13 + [2026-10-12 21:56:27] iteration 510/ 2384 | consumed samples: 16320 | elapsed time per iteration (ms): 247006.6 | throughput per GPU (TFLOP/s/GPU): 18.6 | MFU 5.97% | FLOPs estimator: workload-aware | logical tokens/s: 67922.0 | valid tokens/s: 67922.0 | physical tokens/s: 69061.4 | loss tokens/s: 51239.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.986166E-01 | lm-global-supervised-tokens: 1.265636E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.536961E-02 | lm-token-buffer-mib: 2.387008E+02 | global_load_balancing_loss: 1.000483E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11207.402 | params rms: 0.060684 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:34:50.412004 | finish at 2026-10-18 06:31:17 + [2026-10-12 22:00:08] iteration 511/ 2384 | consumed samples: 16352 | elapsed time per iteration (ms): 221104.5 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.15% | FLOPs estimator: workload-aware | logical tokens/s: 75879.1 | valid tokens/s: 75879.1 | physical tokens/s: 76515.9 | loss tokens/s: 54169.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.122568E-01 | lm-global-supervised-tokens: 1.197706E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.800898E-02 | lm-token-buffer-mib: 2.333760E+02 | global_load_balancing_loss: 1.000819E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11207.356 | params rms: 0.060684 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:02:08.665639 | finish at 2026-10-17 17:02:16 + [2026-10-12 22:03:48] iteration 512/ 2384 | consumed samples: 16384 | elapsed time per iteration (ms): 220562.5 | throughput per GPU (TFLOP/s/GPU): 26.1 | MFU 8.37% | FLOPs estimator: workload-aware | logical tokens/s: 76065.5 | valid tokens/s: 76065.5 | physical tokens/s: 76659.9 | loss tokens/s: 53539.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.384848E-01 | lm-global-supervised-tokens: 1.180873E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.790494E-02 | lm-token-buffer-mib: 2.340044E+02 | global_load_balancing_loss: 1.001927E+00 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11207.308 | params rms: 0.060684 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 18:41:32.972328 | finish at 2026-10-17 16:45:21 + [2026-10-12 22:07:32] iteration 513/ 2384 | consumed samples: 16416 | elapsed time per iteration (ms): 223992.7 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.34% | FLOPs estimator: workload-aware | logical tokens/s: 74900.5 | valid tokens/s: 74900.5 | physical tokens/s: 75578.2 | loss tokens/s: 57021.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.096708E-01 | lm-global-supervised-tokens: 1.277248E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.297661E-02 | lm-token-buffer-mib: 2.347298E+02 | global_load_balancing_loss: 1.001364E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11207.254 | params rms: 0.060684 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:24:50.420397 | finish at 2026-10-17 18:32:23 + [2026-10-12 22:11:16] iteration 514/ 2384 | consumed samples: 16448 | elapsed time per iteration (ms): 223911.8 | throughput per GPU (TFLOP/s/GPU): 19.1 | MFU 6.12% | FLOPs estimator: workload-aware | logical tokens/s: 74927.6 | valid tokens/s: 74927.6 | physical tokens/s: 75881.6 | loss tokens/s: 58664.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.906469E-01 | lm-global-supervised-tokens: 1.313558E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.780306E-02 | lm-token-buffer-mib: 2.360000E+02 | global_load_balancing_loss: 1.002096E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11207.202 | params rms: 0.060683 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:18:35.010543 | finish at 2026-10-17 18:29:51 + [2026-10-12 22:14:57] iteration 515/ 2384 | consumed samples: 16480 | elapsed time per iteration (ms): 220438.9 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.77% | FLOPs estimator: workload-aware | logical tokens/s: 76108.2 | valid tokens/s: 76108.2 | physical tokens/s: 77040.2 | loss tokens/s: 60440.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.998107E-01 | lm-global-supervised-tokens: 1.332347E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.992461E-02 | lm-token-buffer-mib: 2.344453E+02 | global_load_balancing_loss: 1.000127E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11207.157 | params rms: 0.060683 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 18:26:40.364691 | finish at 2026-10-17 16:41:37 + [2026-10-12 22:18:39] iteration 516/ 2384 | consumed samples: 16512 | elapsed time per iteration (ms): 221978.9 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.34% | FLOPs estimator: workload-aware | logical tokens/s: 75580.2 | valid tokens/s: 75580.1 | physical tokens/s: 76411.5 | loss tokens/s: 58445.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.434105E-01 | lm-global-supervised-tokens: 1.297364E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.354340E-02 | lm-token-buffer-mib: 2.347692E+02 | global_load_balancing_loss: 9.997735E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11207.113 | params rms: 0.060683 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:10:56.499607 | finish at 2026-10-17 17:29:35 + [2026-10-12 22:22:18] iteration 517/ 2384 | consumed samples: 16544 | elapsed time per iteration (ms): 219158.6 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.86% | FLOPs estimator: workload-aware | logical tokens/s: 76552.6 | valid tokens/s: 76552.6 | physical tokens/s: 76981.6 | loss tokens/s: 57514.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.276724E-01 | lm-global-supervised-tokens: 1.260482E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.711278E-02 | lm-token-buffer-mib: 2.333017E+02 | global_load_balancing_loss: 1.000012E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11207.066 | params rms: 0.060683 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:39:29.068091 | finish at 2026-10-17 16:01:47 + [2026-10-12 22:26:02] iteration 518/ 2384 | consumed samples: 16576 | elapsed time per iteration (ms): 224104.3 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.88% | FLOPs estimator: workload-aware | logical tokens/s: 74863.4 | valid tokens/s: 74863.4 | physical tokens/s: 75948.9 | loss tokens/s: 53626.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.423633E-01 | lm-global-supervised-tokens: 1.201796E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.392706E-02 | lm-token-buffer-mib: 2.360547E+02 | global_load_balancing_loss: 9.998465E-01 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11207.018 | params rms: 0.060682 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:09:38.655873 | finish at 2026-10-17 18:35:40 + [2026-10-12 22:29:45] iteration 519/ 2384 | consumed samples: 16608 | elapsed time per iteration (ms): 223619.0 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.15% | FLOPs estimator: workload-aware | logical tokens/s: 75025.7 | valid tokens/s: 75025.7 | physical tokens/s: 75749.4 | loss tokens/s: 56649.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.116921E-01 | lm-global-supervised-tokens: 1.266784E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.346421E-02 | lm-token-buffer-mib: 2.331627E+02 | global_load_balancing_loss: 1.000732E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11206.968 | params rms: 0.060682 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:50:49.355774 | finish at 2026-10-17 18:20:35 + [2026-10-12 22:33:49] iteration 520/ 2384 | consumed samples: 16640 | elapsed time per iteration (ms): 243547.4 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.61% | FLOPs estimator: workload-aware | logical tokens/s: 68886.8 | valid tokens/s: 68886.8 | physical tokens/s: 69298.1 | loss tokens/s: 51017.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.091291E-01 | lm-global-supervised-tokens: 1.242518E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.626329E-02 | lm-token-buffer-mib: 2.343984E+02 | global_load_balancing_loss: 1.000955E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11206.916 | params rms: 0.060682 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:06:12.426924 | finish at 2026-10-18 04:40:01 +times across ranks (ms): + save-checkpoint: + rank 0: 25341.94 + rank 1: 25341.92 + rank 2: 25341.93 + rank 3: 25341.91 + rank 4: 25341.90 + rank 5: 25341.92 + rank 6: 25341.96 + rank 7: 25341.91 + rank 8: 25341.25 + rank 9: 25341.46 + rank 10: 25341.49 + rank 11: 25341.39 + rank 12: 25341.41 + rank 13: 25341.39 + rank 14: 25341.46 + rank 15: 25341.41 + rank 16: 25341.91 + rank 17: 25341.90 + rank 18: 25341.91 + rank 19: 25341.91 + rank 20: 25341.87 + rank 21: 25341.91 + rank 22: 25341.96 + rank 23: 25341.87 + rank 24: 25341.80 + rank 25: 25341.93 + rank 26: 25341.88 + rank 27: 25341.80 + rank 28: 25341.90 + rank 29: 25341.94 + rank 30: 25341.89 + rank 31: 25341.81 + rank 32: 25341.46 + rank 33: 25341.52 + rank 34: 25341.45 + rank 35: 25341.47 + rank 36: 25341.48 + rank 37: 25341.45 + rank 38: 25341.47 + rank 39: 25341.48 + rank 40: 25341.59 + rank 41: 25341.59 + rank 42: 25341.58 + rank 43: 25341.60 + rank 44: 25341.62 + rank 45: 25341.56 + rank 46: 25341.55 + rank 47: 25341.57 + rank 48: 25341.82 + rank 49: 25341.87 + rank 50: 25341.84 + rank 51: 25341.78 + rank 52: 25341.77 + rank 53: 25341.78 + rank 54: 25341.78 + rank 55: 25341.80 + rank 56: 25341.81 + rank 57: 25341.88 + rank 58: 25341.82 + rank 59: 25341.84 + rank 60: 25341.85 + rank 61: 25341.84 + rank 62: 25341.82 + rank 63: 25341.81 + [2026-10-12 22:37:56] iteration 521/ 2384 | consumed samples: 16672 | elapsed time per iteration (ms): 221926.2 | throughput per GPU (TFLOP/s/GPU): 19.5 | MFU 6.24% | FLOPs estimator: workload-aware | logical tokens/s: 75598.1 | valid tokens/s: 75598.1 | physical tokens/s: 76587.9 | loss tokens/s: 58895.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.498210E-01 | lm-global-supervised-tokens: 1.307045E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.336671E-01 | lm-token-buffer-mib: 2.354704E+02 | global_load_balancing_loss: 1.000759E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11206.863 | params rms: 0.060682 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 18:50:48.447809 | finish at 2026-10-17 17:28:45 + [2026-10-12 22:41:41] iteration 522/ 2384 | consumed samples: 16704 | elapsed time per iteration (ms): 224898.3 | throughput per GPU (TFLOP/s/GPU): 18.1 | MFU 5.80% | FLOPs estimator: workload-aware | logical tokens/s: 74599.0 | valid tokens/s: 74599.0 | physical tokens/s: 75187.3 | loss tokens/s: 56535.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.649239E-01 | lm-global-supervised-tokens: 1.271476E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.830069E-02 | lm-token-buffer-mib: 2.332015E+02 | global_load_balancing_loss: 1.000018E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11206.814 | params rms: 0.060681 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:19:20.712607 | finish at 2026-10-17 19:01:02 + [2026-10-12 22:45:20] iteration 523/ 2384 | consumed samples: 16736 | elapsed time per iteration (ms): 219106.1 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.41% | FLOPs estimator: workload-aware | logical tokens/s: 76571.1 | valid tokens/s: 76571.1 | physical tokens/s: 77575.1 | loss tokens/s: 58756.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.348522E-01 | lm-global-supervised-tokens: 1.287387E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.969560E-02 | lm-token-buffer-mib: 2.342439E+02 | global_load_balancing_loss: 9.996264E-01 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11206.767 | params rms: 0.060681 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:15:56.488636 | finish at 2026-10-17 16:01:17 + [2026-10-12 22:49:02] iteration 524/ 2384 | consumed samples: 16768 | elapsed time per iteration (ms): 221576.5 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.97% | FLOPs estimator: workload-aware | logical tokens/s: 75717.3 | valid tokens/s: 75717.3 | physical tokens/s: 77119.5 | loss tokens/s: 58467.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.114246E-01 | lm-global-supervised-tokens: 1.295499E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.368531E-02 | lm-token-buffer-mib: 2.342901E+02 | global_load_balancing_loss: 9.998070E-01 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11206.719 | params rms: 0.060681 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 18:28:52.226028 | finish at 2026-10-17 17:17:54 + [2026-10-12 22:52:43] iteration 525/ 2384 | consumed samples: 16800 | elapsed time per iteration (ms): 221346.8 | throughput per GPU (TFLOP/s/GPU): 24.4 | MFU 7.83% | FLOPs estimator: workload-aware | logical tokens/s: 75795.9 | valid tokens/s: 75795.9 | physical tokens/s: 76501.0 | loss tokens/s: 58876.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.492880E-01 | lm-global-supervised-tokens: 1.303215E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.201778E-02 | lm-token-buffer-mib: 2.339994E+02 | global_load_balancing_loss: 1.001479E+00 | loss scale: 1.0 | grad norm: 0.157 | params norm: 11206.671 | params rms: 0.060680 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 18:18:03.775826 | finish at 2026-10-17 17:10:47 + [2026-10-12 22:56:33] iteration 526/ 2384 | consumed samples: 16832 | elapsed time per iteration (ms): 230016.3 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.19% | FLOPs estimator: workload-aware | logical tokens/s: 72939.1 | valid tokens/s: 72939.1 | physical tokens/s: 74083.1 | loss tokens/s: 55452.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.857777E-01 | lm-global-supervised-tokens: 1.275493E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.048927E-02 | lm-token-buffer-mib: 2.370938E+02 | global_load_balancing_loss: 9.993433E-01 | loss scale: 1.0 | grad norm: 0.192 | params norm: 11206.626 | params rms: 0.060680 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:42:50.296406 | finish at 2026-10-17 21:39:24 + [2026-10-12 23:00:19] iteration 527/ 2384 | consumed samples: 16864 | elapsed time per iteration (ms): 225835.3 | throughput per GPU (TFLOP/s/GPU): 26.1 | MFU 8.35% | FLOPs estimator: workload-aware | logical tokens/s: 74289.4 | valid tokens/s: 74289.4 | physical tokens/s: 75049.1 | loss tokens/s: 55936.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.560523E-01 | lm-global-supervised-tokens: 1.263250E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.400226E-02 | lm-token-buffer-mib: 2.335589E+02 | global_load_balancing_loss: 1.000697E+00 | loss scale: 1.0 | grad norm: 0.117 | params norm: 11206.580 | params rms: 0.060680 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:29:36.159126 | finish at 2026-10-17 19:29:55 + [2026-10-12 23:04:02] iteration 528/ 2384 | consumed samples: 16896 | elapsed time per iteration (ms): 223031.4 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.24% | FLOPs estimator: workload-aware | logical tokens/s: 75223.5 | valid tokens/s: 75223.5 | physical tokens/s: 75852.3 | loss tokens/s: 58006.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.989670E-01 | lm-global-supervised-tokens: 1.293729E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.412241E-02 | lm-token-buffer-mib: 2.327823E+02 | global_load_balancing_loss: 1.001035E+00 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11206.530 | params rms: 0.060680 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 18:59:06.305328 | finish at 2026-10-17 18:03:08 + [2026-10-12 23:07:45] iteration 529/ 2384 | consumed samples: 16928 | elapsed time per iteration (ms): 223107.7 | throughput per GPU (TFLOP/s/GPU): 18.8 | MFU 6.02% | FLOPs estimator: workload-aware | logical tokens/s: 75197.8 | valid tokens/s: 75197.8 | physical tokens/s: 76157.9 | loss tokens/s: 56170.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.913446E-01 | lm-global-supervised-tokens: 1.253210E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.474439E-02 | lm-token-buffer-mib: 2.350242E+02 | global_load_balancing_loss: 1.000469E+00 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11206.476 | params rms: 0.060679 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 18:57:44.869061 | finish at 2026-10-17 18:05:30 + [2026-10-12 23:11:49] iteration 530/ 2384 | consumed samples: 16960 | elapsed time per iteration (ms): 243366.5 | throughput per GPU (TFLOP/s/GPU): 19.5 | MFU 6.24% | FLOPs estimator: workload-aware | logical tokens/s: 68937.9 | valid tokens/s: 68937.9 | physical tokens/s: 69556.3 | loss tokens/s: 53975.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.686459E-01 | lm-global-supervised-tokens: 1.313579E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.316258E-02 | lm-token-buffer-mib: 2.336134E+02 | global_load_balancing_loss: 1.001421E+00 | loss scale: 1.0 | grad norm: 0.190 | params norm: 11206.424 | params rms: 0.060679 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:20:01.560657 | finish at 2026-10-18 04:31:50 + [2026-10-12 23:15:32] iteration 531/ 2384 | consumed samples: 16992 | elapsed time per iteration (ms): 223475.3 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.52% | FLOPs estimator: workload-aware | logical tokens/s: 75074.0 | valid tokens/s: 75074.0 | physical tokens/s: 76204.5 | loss tokens/s: 57337.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.276588E-01 | lm-global-supervised-tokens: 1.281359E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.908812E-02 | lm-token-buffer-mib: 2.352258E+02 | global_load_balancing_loss: 1.000230E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11206.372 | params rms: 0.060679 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:01:39.817627 | finish at 2026-10-17 18:17:12 + [2026-10-12 23:19:15] iteration 532/ 2384 | consumed samples: 17024 | elapsed time per iteration (ms): 223307.0 | throughput per GPU (TFLOP/s/GPU): 24.3 | MFU 7.77% | FLOPs estimator: workload-aware | logical tokens/s: 75130.5 | valid tokens/s: 75130.5 | physical tokens/s: 75991.1 | loss tokens/s: 51776.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.340519E-01 | lm-global-supervised-tokens: 1.156214E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.313534E-02 | lm-token-buffer-mib: 2.332983E+02 | global_load_balancing_loss: 1.001107E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11206.319 | params rms: 0.060679 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 18:52:44.494090 | finish at 2026-10-17 18:12:00 + [2026-10-12 23:23:00] iteration 533/ 2384 | consumed samples: 17056 | elapsed time per iteration (ms): 224920.9 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.05% | FLOPs estimator: workload-aware | logical tokens/s: 74591.5 | valid tokens/s: 74591.5 | physical tokens/s: 75479.4 | loss tokens/s: 60639.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.372313E-01 | lm-global-supervised-tokens: 1.363918E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.349945E-02 | lm-token-buffer-mib: 2.343631E+02 | global_load_balancing_loss: 1.000593E+00 | loss scale: 1.0 | grad norm: 0.188 | params norm: 11206.268 | params rms: 0.060678 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:38:48.562707 | finish at 2026-10-17 19:01:49 + [2026-10-12 23:26:45] iteration 534/ 2384 | consumed samples: 17088 | elapsed time per iteration (ms): 224743.8 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.90% | FLOPs estimator: workload-aware | logical tokens/s: 74650.3 | valid tokens/s: 74650.3 | physical tokens/s: 75477.4 | loss tokens/s: 56131.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.324072E-01 | lm-global-supervised-tokens: 1.261528E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.291697E-02 | lm-token-buffer-mib: 2.353121E+02 | global_load_balancing_loss: 1.000595E+00 | loss scale: 1.0 | grad norm: 0.234 | params norm: 11206.217 | params rms: 0.060678 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:29:36.037800 | finish at 2026-10-17 18:56:21 + [2026-10-12 23:30:27] iteration 535/ 2384 | consumed samples: 17120 | elapsed time per iteration (ms): 221993.3 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.79% | FLOPs estimator: workload-aware | logical tokens/s: 75575.1 | valid tokens/s: 75575.1 | physical tokens/s: 76569.9 | loss tokens/s: 58185.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.913007E-01 | lm-global-supervised-tokens: 1.291683E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.280537E-02 | lm-token-buffer-mib: 2.343457E+02 | global_load_balancing_loss: 1.000872E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11206.166 | params rms: 0.060678 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 18:01:05.690538 | finish at 2026-10-17 17:31:33 + [2026-10-12 23:34:08] iteration 536/ 2384 | consumed samples: 17152 | elapsed time per iteration (ms): 220913.0 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.41% | FLOPs estimator: workload-aware | logical tokens/s: 75944.9 | valid tokens/s: 75944.9 | physical tokens/s: 76213.7 | loss tokens/s: 52908.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.056991E-01 | lm-global-supervised-tokens: 1.168826E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.289793E-02 | lm-token-buffer-mib: 2.332700E+02 | global_load_balancing_loss: 9.999356E-01 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11206.116 | params rms: 0.060677 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:24:07.227722 | finish at 2026-10-17 16:58:15 + [2026-10-12 23:37:48] iteration 537/ 2384 | consumed samples: 17184 | elapsed time per iteration (ms): 219740.6 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.34% | FLOPs estimator: workload-aware | logical tokens/s: 76349.9 | valid tokens/s: 76349.9 | physical tokens/s: 76900.3 | loss tokens/s: 57513.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.233408E-01 | lm-global-supervised-tokens: 1.263803E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.516937E-02 | lm-token-buffer-mib: 2.339165E+02 | global_load_balancing_loss: 1.000194E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11206.065 | params rms: 0.060677 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:44:20.819265 | finish at 2026-10-17 16:22:08 + [2026-10-12 23:41:34] iteration 538/ 2384 | consumed samples: 17216 | elapsed time per iteration (ms): 225951.5 | throughput per GPU (TFLOP/s/GPU): 25.7 | MFU 8.25% | FLOPs estimator: workload-aware | logical tokens/s: 74251.3 | valid tokens/s: 74251.3 | physical tokens/s: 75233.4 | loss tokens/s: 62957.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.365298E-01 | lm-global-supervised-tokens: 1.422529E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.641535E-02 | lm-token-buffer-mib: 2.325475E+02 | global_load_balancing_loss: 9.998648E-01 | loss scale: 1.0 | grad norm: 0.177 | params norm: 11206.014 | params rms: 0.060677 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:51:46.478130 | finish at 2026-10-17 19:33:20 + [2026-10-12 23:45:16] iteration 539/ 2384 | consumed samples: 17248 | elapsed time per iteration (ms): 222292.0 | throughput per GPU (TFLOP/s/GPU): 20.2 | MFU 6.48% | FLOPs estimator: workload-aware | logical tokens/s: 75473.6 | valid tokens/s: 75473.6 | physical tokens/s: 76903.8 | loss tokens/s: 57167.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.238246E-01 | lm-global-supervised-tokens: 1.270792E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.772165E-02 | lm-token-buffer-mib: 2.365138E+02 | global_load_balancing_loss: 1.000431E+00 | loss scale: 1.0 | grad norm: 0.147 | params norm: 11205.965 | params rms: 0.060677 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:55:28.741862 | finish at 2026-10-17 17:40:45 + [2026-10-12 23:49:23] iteration 540/ 2384 | consumed samples: 17280 | elapsed time per iteration (ms): 247019.8 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.82% | FLOPs estimator: workload-aware | logical tokens/s: 67918.4 | valid tokens/s: 67918.4 | physical tokens/s: 68234.3 | loss tokens/s: 53331.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.037701E-01 | lm-global-supervised-tokens: 1.317397E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.460850E-02 | lm-token-buffer-mib: 2.330017E+02 | global_load_balancing_loss: 9.991301E-01 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11205.916 | params rms: 0.060676 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:31:44.479010 | finish at 2026-10-18 06:21:07 + [2026-10-12 23:53:10] iteration 541/ 2384 | consumed samples: 17312 | elapsed time per iteration (ms): 226925.6 | throughput per GPU (TFLOP/s/GPU): 24.0 | MFU 7.70% | FLOPs estimator: workload-aware | logical tokens/s: 73932.6 | valid tokens/s: 73932.6 | physical tokens/s: 74399.7 | loss tokens/s: 58169.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.877312E-01 | lm-global-supervised-tokens: 1.320010E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.683607E-02 | lm-token-buffer-mib: 2.337157E+02 | global_load_balancing_loss: 9.998785E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11205.871 | params rms: 0.060676 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:10:23.851455 | finish at 2026-10-17 20:03:34 + [2026-10-12 23:56:50] iteration 542/ 2384 | consumed samples: 17344 | elapsed time per iteration (ms): 220462.5 | throughput per GPU (TFLOP/s/GPU): 18.6 | MFU 5.98% | FLOPs estimator: workload-aware | logical tokens/s: 76100.0 | valid tokens/s: 76100.0 | physical tokens/s: 76693.7 | loss tokens/s: 57440.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.406320E-01 | lm-global-supervised-tokens: 1.266354E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.881115E-02 | lm-token-buffer-mib: 2.336283E+02 | global_load_balancing_loss: 1.000719E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11205.821 | params rms: 0.060676 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:48:11.946695 | finish at 2026-10-17 16:45:02 + [2026-10-13 00:00:32] iteration 543/ 2384 | consumed samples: 17376 | elapsed time per iteration (ms): 221877.7 | throughput per GPU (TFLOP/s/GPU): 24.7 | MFU 7.93% | FLOPs estimator: workload-aware | logical tokens/s: 75614.4 | valid tokens/s: 75614.4 | physical tokens/s: 76448.3 | loss tokens/s: 59010.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.294773E-01 | lm-global-supervised-tokens: 1.309317E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.271645E-02 | lm-token-buffer-mib: 2.354771E+02 | global_load_balancing_loss: 1.002396E+00 | loss scale: 1.0 | grad norm: 0.129 | params norm: 11205.770 | params rms: 0.060676 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:27:56.901610 | finish at 2026-10-17 17:28:29 + [2026-10-13 00:04:14] iteration 544/ 2384 | consumed samples: 17408 | elapsed time per iteration (ms): 221787.2 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.69% | FLOPs estimator: workload-aware | logical tokens/s: 75645.5 | valid tokens/s: 75645.5 | physical tokens/s: 76555.4 | loss tokens/s: 59162.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.189779E-01 | lm-global-supervised-tokens: 1.312146E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.469428E-02 | lm-token-buffer-mib: 2.339407E+02 | global_load_balancing_loss: 1.001728E+00 | loss scale: 1.0 | grad norm: 0.154 | params norm: 11205.719 | params rms: 0.060675 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:21:28.502789 | finish at 2026-10-17 17:25:42 + [2026-10-13 00:07:55] iteration 545/ 2384 | consumed samples: 17440 | elapsed time per iteration (ms): 221186.5 | throughput per GPU (TFLOP/s/GPU): 23.7 | MFU 7.58% | FLOPs estimator: workload-aware | logical tokens/s: 75850.9 | valid tokens/s: 75850.9 | physical tokens/s: 76329.6 | loss tokens/s: 59121.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.992651E-01 | lm-global-supervised-tokens: 1.307696E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.794073E-02 | lm-token-buffer-mib: 2.338007E+02 | global_load_balancing_loss: 1.001729E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11205.673 | params rms: 0.060675 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:59:22.024932 | finish at 2026-10-17 17:07:17 + [2026-10-13 00:11:41] iteration 546/ 2384 | consumed samples: 17472 | elapsed time per iteration (ms): 226254.2 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.43% | FLOPs estimator: workload-aware | logical tokens/s: 74152.0 | valid tokens/s: 74152.0 | physical tokens/s: 75231.8 | loss tokens/s: 55441.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.032297E-01 | lm-global-supervised-tokens: 1.254393E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.751399E-02 | lm-token-buffer-mib: 2.347231E+02 | global_load_balancing_loss: 9.996946E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11205.632 | params rms: 0.060675 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:30:55.202038 | finish at 2026-10-17 19:42:37 + [2026-10-13 00:15:27] iteration 547/ 2384 | consumed samples: 17504 | elapsed time per iteration (ms): 225507.7 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.20% | FLOPs estimator: workload-aware | logical tokens/s: 74397.3 | valid tokens/s: 74397.3 | physical tokens/s: 75148.5 | loss tokens/s: 52952.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.529186E-01 | lm-global-supervised-tokens: 1.194122E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.315633E-02 | lm-token-buffer-mib: 2.348485E+02 | global_load_balancing_loss: 9.992157E-01 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11205.590 | params rms: 0.060675 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:04:17.659729 | finish at 2026-10-17 19:19:45 + [2026-10-13 00:19:08] iteration 548/ 2384 | consumed samples: 17536 | elapsed time per iteration (ms): 221258.2 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.71% | FLOPs estimator: workload-aware | logical tokens/s: 75826.3 | valid tokens/s: 75826.3 | physical tokens/s: 76600.2 | loss tokens/s: 61736.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.932269E-01 | lm-global-supervised-tokens: 1.365962E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 2.705901E-01 | lm-token-buffer-mib: 2.341475E+02 | global_load_balancing_loss: 9.997584E-01 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11205.546 | params rms: 0.060674 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:50:30.099721 | finish at 2026-10-17 17:09:38 + [2026-10-13 00:22:51] iteration 549/ 2384 | consumed samples: 17568 | elapsed time per iteration (ms): 222642.6 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.37% | FLOPs estimator: workload-aware | logical tokens/s: 75354.7 | valid tokens/s: 75354.7 | physical tokens/s: 75745.3 | loss tokens/s: 55716.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.482076E-01 | lm-global-supervised-tokens: 1.240486E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.480363E-02 | lm-token-buffer-mib: 2.332019E+02 | global_load_balancing_loss: 9.994490E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11205.507 | params rms: 0.060674 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:29:09.084047 | finish at 2026-10-17 17:52:00 + [2026-10-13 00:26:56] iteration 550/ 2384 | consumed samples: 17600 | elapsed time per iteration (ms): 244689.8 | throughput per GPU (TFLOP/s/GPU): 19.4 | MFU 6.22% | FLOPs estimator: workload-aware | logical tokens/s: 68565.2 | valid tokens/s: 68565.2 | physical tokens/s: 69478.9 | loss tokens/s: 50400.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.747700E-01 | lm-global-supervised-tokens: 1.233237E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.329032E-02 | lm-token-buffer-mib: 2.341783E+02 | global_load_balancing_loss: 1.000277E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11205.466 | params rms: 0.060674 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:39:21.016724 | finish at 2026-10-18 05:06:17 + [2026-10-13 00:30:38] iteration 551/ 2384 | consumed samples: 17632 | elapsed time per iteration (ms): 222566.8 | throughput per GPU (TFLOP/s/GPU): 23.8 | MFU 7.62% | FLOPs estimator: workload-aware | logical tokens/s: 75380.5 | valid tokens/s: 75380.5 | physical tokens/s: 76284.9 | loss tokens/s: 58658.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.062754E-01 | lm-global-supervised-tokens: 1.305550E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.622220E-02 | lm-token-buffer-mib: 2.332708E+02 | global_load_balancing_loss: 1.000397E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11205.427 | params rms: 0.060674 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:19:24.973896 | finish at 2026-10-17 17:50:03 + [2026-10-13 00:34:18] iteration 552/ 2384 | consumed samples: 17664 | elapsed time per iteration (ms): 220211.9 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.39% | FLOPs estimator: workload-aware | logical tokens/s: 76186.5 | valid tokens/s: 76186.5 | physical tokens/s: 76497.7 | loss tokens/s: 60544.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.382173E-01 | lm-global-supervised-tokens: 1.333271E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.819158E-02 | lm-token-buffer-mib: 2.329470E+02 | global_load_balancing_loss: 1.000807E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11205.386 | params rms: 0.060674 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:03:48.188562 | finish at 2026-10-17 16:38:07 + [2026-10-13 00:38:00] iteration 553/ 2384 | consumed samples: 17696 | elapsed time per iteration (ms): 221222.8 | throughput per GPU (TFLOP/s/GPU): 26.5 | MFU 8.51% | FLOPs estimator: workload-aware | logical tokens/s: 75838.5 | valid tokens/s: 75838.5 | physical tokens/s: 76047.1 | loss tokens/s: 56027.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.101289E-01 | lm-global-supervised-tokens: 1.239466E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.267823E-02 | lm-token-buffer-mib: 2.322669E+02 | global_load_balancing_loss: 1.000680E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11205.341 | params rms: 0.060673 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:30:58.874364 | finish at 2026-10-17 17:08:58 + [2026-10-13 00:41:46] iteration 554/ 2384 | consumed samples: 17728 | elapsed time per iteration (ms): 226851.1 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.57% | FLOPs estimator: workload-aware | logical tokens/s: 73956.8 | valid tokens/s: 73956.8 | physical tokens/s: 75307.7 | loss tokens/s: 53624.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.229896E-01 | lm-global-supervised-tokens: 1.216468E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.614793E-02 | lm-token-buffer-mib: 2.392523E+02 | global_load_balancing_loss: 1.000278E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11205.296 | params rms: 0.060673 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:18:57.568352 | finish at 2026-10-17 20:00:44 + [2026-10-13 00:45:30] iteration 555/ 2384 | consumed samples: 17760 | elapsed time per iteration (ms): 223458.9 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.57% | FLOPs estimator: workload-aware | logical tokens/s: 75079.6 | valid tokens/s: 75079.6 | physical tokens/s: 75855.2 | loss tokens/s: 60095.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.815812E-01 | lm-global-supervised-tokens: 1.342876E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.521943E-02 | lm-token-buffer-mib: 2.343006E+02 | global_load_balancing_loss: 1.000189E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11205.257 | params rms: 0.060673 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:31:46.414395 | finish at 2026-10-17 18:17:16 + [2026-10-13 00:49:14] iteration 556/ 2384 | consumed samples: 17792 | elapsed time per iteration (ms): 224173.6 | throughput per GPU (TFLOP/s/GPU): 19.0 | MFU 6.08% | FLOPs estimator: workload-aware | logical tokens/s: 74840.3 | valid tokens/s: 74840.3 | physical tokens/s: 75960.0 | loss tokens/s: 53485.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.924599E-01 | lm-global-supervised-tokens: 1.199001E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.410597E-02 | lm-token-buffer-mib: 2.373657E+02 | global_load_balancing_loss: 1.000669E+00 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11205.219 | params rms: 0.060673 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:49:49.368181 | finish at 2026-10-17 18:39:03 + [2026-10-13 00:52:58] iteration 557/ 2384 | consumed samples: 17824 | elapsed time per iteration (ms): 223877.7 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.93% | FLOPs estimator: workload-aware | logical tokens/s: 74939.0 | valid tokens/s: 74939.0 | physical tokens/s: 75596.7 | loss tokens/s: 56528.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.712556E-01 | lm-global-supervised-tokens: 1.265554E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.503459E-02 | lm-token-buffer-mib: 2.332113E+02 | global_load_balancing_loss: 1.001681E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11205.182 | params rms: 0.060672 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:37:04.590299 | finish at 2026-10-17 18:30:02 + [2026-10-13 00:56:39] iteration 558/ 2384 | consumed samples: 17856 | elapsed time per iteration (ms): 220835.7 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.82% | FLOPs estimator: workload-aware | logical tokens/s: 75971.4 | valid tokens/s: 75971.4 | physical tokens/s: 76636.3 | loss tokens/s: 60658.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.929106E-01 | lm-global-supervised-tokens: 1.339554E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.312152E-02 | lm-token-buffer-mib: 2.330170E+02 | global_load_balancing_loss: 1.000017E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11205.147 | params rms: 0.060672 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:00:46.058356 | finish at 2026-10-17 16:57:25 + [2026-10-13 01:00:20] iteration 559/ 2384 | consumed samples: 17888 | elapsed time per iteration (ms): 221368.9 | throughput per GPU (TFLOP/s/GPU): 27.1 | MFU 8.70% | FLOPs estimator: workload-aware | logical tokens/s: 75788.4 | valid tokens/s: 75788.4 | physical tokens/s: 76048.7 | loss tokens/s: 60297.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.523737E-01 | lm-global-supervised-tokens: 1.334808E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.590396E-02 | lm-token-buffer-mib: 2.332461E+02 | global_load_balancing_loss: 1.000583E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11205.110 | params rms: 0.060672 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:13:18.223031 | finish at 2026-10-17 17:13:38 + [2026-10-13 01:04:29] iteration 560/ 2384 | consumed samples: 17920 | elapsed time per iteration (ms): 249128.1 | throughput per GPU (TFLOP/s/GPU): 19.9 | MFU 6.38% | FLOPs estimator: workload-aware | logical tokens/s: 67343.7 | valid tokens/s: 67343.7 | physical tokens/s: 68306.8 | loss tokens/s: 52091.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.545981E-01 | lm-global-supervised-tokens: 1.297757E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.554669E-02 | lm-token-buffer-mib: 2.362462E+02 | global_load_balancing_loss: 1.001030E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11205.072 | params rms: 0.060672 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:13:29.733833 | finish at 2026-10-18 07:17:59 +times across ranks (ms): + save-checkpoint: + rank 0: 25292.67 + rank 1: 25292.69 + rank 2: 25292.69 + rank 3: 25292.66 + rank 4: 25292.65 + rank 5: 25292.66 + rank 6: 25292.67 + rank 7: 25292.67 + rank 8: 25292.87 + rank 9: 25292.92 + rank 10: 25292.92 + rank 11: 25292.87 + rank 12: 25292.90 + rank 13: 25292.87 + rank 14: 25292.88 + rank 15: 25292.88 + rank 16: 25292.63 + rank 17: 25292.67 + rank 18: 25292.69 + rank 19: 25292.67 + rank 20: 25292.63 + rank 21: 25292.63 + rank 22: 25292.70 + rank 23: 25292.62 + rank 24: 25292.64 + rank 25: 25292.67 + rank 26: 25292.62 + rank 27: 25292.61 + rank 28: 25292.63 + rank 29: 25292.69 + rank 30: 25292.39 + rank 31: 25292.66 + rank 32: 25292.85 + rank 33: 25292.71 + rank 34: 25292.86 + rank 35: 25292.85 + rank 36: 25292.80 + rank 37: 25292.84 + rank 38: 25292.82 + rank 39: 25292.99 + rank 40: 25291.78 + rank 41: 25291.77 + rank 42: 25291.77 + rank 43: 25291.80 + rank 44: 25291.78 + rank 45: 25291.77 + rank 46: 25291.76 + rank 47: 25291.80 + rank 48: 25293.21 + rank 49: 25293.12 + rank 50: 25293.19 + rank 51: 25293.16 + rank 52: 25293.03 + rank 53: 25293.16 + rank 54: 25293.14 + rank 55: 25293.09 + rank 56: 25292.36 + rank 57: 25292.40 + rank 58: 25292.38 + rank 59: 25292.39 + rank 60: 25292.40 + rank 61: 25292.38 + rank 62: 25292.36 + rank 63: 25292.36 + [2026-10-13 01:08:42] iteration 561/ 2384 | consumed samples: 17952 | elapsed time per iteration (ms): 227440.9 | throughput per GPU (TFLOP/s/GPU): 19.9 | MFU 6.39% | FLOPs estimator: workload-aware | logical tokens/s: 73765.1 | valid tokens/s: 73765.1 | physical tokens/s: 74564.3 | loss tokens/s: 58994.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.665864E-01 | lm-global-supervised-tokens: 1.341785E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.759065E-02 | lm-token-buffer-mib: 2.358089E+02 | global_load_balancing_loss: 9.995201E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11205.034 | params rms: 0.060672 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:10:24.691317 | finish at 2026-10-17 20:19:07 + [2026-10-13 01:12:32] iteration 562/ 2384 | consumed samples: 17984 | elapsed time per iteration (ms): 229709.3 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.56% | FLOPs estimator: workload-aware | logical tokens/s: 73036.5 | valid tokens/s: 73036.5 | physical tokens/s: 73629.9 | loss tokens/s: 56446.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.084627E-01 | lm-global-supervised-tokens: 1.296617E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 2.631398E-01 | lm-token-buffer-mib: 2.332873E+02 | global_load_balancing_loss: 9.997153E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11204.995 | params rms: 0.060671 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:15:30.276040 | finish at 2026-10-17 21:28:02 + [2026-10-13 01:16:15] iteration 563/ 2384 | consumed samples: 18016 | elapsed time per iteration (ms): 222771.0 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.09% | FLOPs estimator: workload-aware | logical tokens/s: 75311.3 | valid tokens/s: 75311.3 | physical tokens/s: 75801.9 | loss tokens/s: 55090.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.641079E-01 | lm-global-supervised-tokens: 1.227248E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.709765E-02 | lm-token-buffer-mib: 2.336417E+02 | global_load_balancing_loss: 9.999418E-01 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11204.956 | params rms: 0.060671 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:41:05.988228 | finish at 2026-10-17 17:57:20 + [2026-10-13 01:19:56] iteration 564/ 2384 | consumed samples: 18048 | elapsed time per iteration (ms): 221902.3 | throughput per GPU (TFLOP/s/GPU): 18.6 | MFU 5.97% | FLOPs estimator: workload-aware | logical tokens/s: 75606.3 | valid tokens/s: 75606.3 | physical tokens/s: 76286.1 | loss tokens/s: 57511.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.898023E-01 | lm-global-supervised-tokens: 1.276183E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.231941E-02 | lm-token-buffer-mib: 2.348663E+02 | global_load_balancing_loss: 1.000415E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11204.916 | params rms: 0.060671 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:11:02.195330 | finish at 2026-10-17 17:30:59 + [2026-10-13 01:23:37] iteration 565/ 2384 | consumed samples: 18080 | elapsed time per iteration (ms): 221054.5 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.39% | FLOPs estimator: workload-aware | logical tokens/s: 75896.3 | valid tokens/s: 75896.3 | physical tokens/s: 76508.1 | loss tokens/s: 58863.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.538867E-01 | lm-global-supervised-tokens: 1.301212E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.253169E-02 | lm-token-buffer-mib: 2.345315E+02 | global_load_balancing_loss: 9.999915E-01 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11204.876 | params rms: 0.060671 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:41:38.107932 | finish at 2026-10-17 17:05:16 + [2026-10-13 01:27:21] iteration 566/ 2384 | consumed samples: 18112 | elapsed time per iteration (ms): 223800.0 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.09% | FLOPs estimator: workload-aware | logical tokens/s: 74965.1 | valid tokens/s: 74965.1 | physical tokens/s: 75447.4 | loss tokens/s: 54050.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.183566E-01 | lm-global-supervised-tokens: 1.209660E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.678100E-02 | lm-token-buffer-mib: 2.336184E+02 | global_load_balancing_loss: 1.001861E+00 | loss scale: 1.0 | grad norm: 0.302 | params norm: 11204.833 | params rms: 0.060671 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:01:08.463630 | finish at 2026-10-17 18:28:30 + [2026-10-13 01:31:05] iteration 567/ 2384 | consumed samples: 18144 | elapsed time per iteration (ms): 223495.3 | throughput per GPU (TFLOP/s/GPU): 19.7 | MFU 6.31% | FLOPs estimator: workload-aware | logical tokens/s: 75067.3 | valid tokens/s: 75067.3 | physical tokens/s: 76084.3 | loss tokens/s: 59748.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.294975E-01 | lm-global-supervised-tokens: 1.335356E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.347149E-02 | lm-token-buffer-mib: 2.343434E+02 | global_load_balancing_loss: 1.000733E+00 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11204.784 | params rms: 0.060670 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:48:11.042075 | finish at 2026-10-17 18:19:16 + [2026-10-13 01:34:53] iteration 568/ 2384 | consumed samples: 18176 | elapsed time per iteration (ms): 228103.1 | throughput per GPU (TFLOP/s/GPU): 18.6 | MFU 5.96% | FLOPs estimator: workload-aware | logical tokens/s: 73550.9 | valid tokens/s: 73550.9 | physical tokens/s: 74524.6 | loss tokens/s: 54556.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.099010E-01 | lm-global-supervised-tokens: 1.244454E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.618010E-02 | lm-token-buffer-mib: 2.363521E+02 | global_load_balancing_loss: 1.001177E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11204.744 | params rms: 0.060670 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:03:55.306347 | finish at 2026-10-17 20:38:48 + [2026-10-13 01:38:36] iteration 569/ 2384 | consumed samples: 18208 | elapsed time per iteration (ms): 222846.0 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.28% | FLOPs estimator: workload-aware | logical tokens/s: 75286.0 | valid tokens/s: 75286.0 | physical tokens/s: 75843.9 | loss tokens/s: 54352.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.961374E-01 | lm-global-supervised-tokens: 1.211229E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.898095E-02 | lm-token-buffer-mib: 2.341450E+02 | global_load_balancing_loss: 9.992638E-01 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11204.710 | params rms: 0.060670 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:21:05.440589 | finish at 2026-10-17 17:59:41 + [2026-10-13 01:42:39] iteration 570/ 2384 | consumed samples: 18240 | elapsed time per iteration (ms): 243602.1 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.36% | FLOPs estimator: workload-aware | logical tokens/s: 68871.2 | valid tokens/s: 68871.2 | physical tokens/s: 69593.1 | loss tokens/s: 53737.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.816777E-01 | lm-global-supervised-tokens: 1.309053E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.433568E-02 | lm-token-buffer-mib: 2.348718E+02 | global_load_balancing_loss: 9.990055E-01 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11204.677 | params rms: 0.060670 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:44:54.160339 | finish at 2026-10-18 04:27:33 + [2026-10-13 01:46:22] iteration 571/ 2384 | consumed samples: 18272 | elapsed time per iteration (ms): 223040.8 | throughput per GPU (TFLOP/s/GPU): 25.9 | MFU 8.30% | FLOPs estimator: workload-aware | logical tokens/s: 75220.0 | valid tokens/s: 75220.0 | physical tokens/s: 75437.6 | loss tokens/s: 56620.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.270092E-01 | lm-global-supervised-tokens: 1.262864E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.493080E-02 | lm-token-buffer-mib: 2.325663E+02 | global_load_balancing_loss: 9.992083E-01 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11204.644 | params rms: 0.060670 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:19:33.037571 | finish at 2026-10-17 18:05:55 + [2026-10-13 01:50:06] iteration 572/ 2384 | consumed samples: 18304 | elapsed time per iteration (ms): 223184.2 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.92% | FLOPs estimator: workload-aware | logical tokens/s: 75172.0 | valid tokens/s: 75172.0 | physical tokens/s: 76302.5 | loss tokens/s: 57416.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.110172E-01 | lm-global-supervised-tokens: 1.281440E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.889959E-02 | lm-token-buffer-mib: 2.345185E+02 | global_load_balancing_loss: 1.002293E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11204.613 | params rms: 0.060669 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:20:09.743703 | finish at 2026-10-17 18:10:15 + [2026-10-13 01:53:50] iteration 573/ 2384 | consumed samples: 18336 | elapsed time per iteration (ms): 224909.0 | throughput per GPU (TFLOP/s/GPU): 19.0 | MFU 6.08% | FLOPs estimator: workload-aware | logical tokens/s: 74595.4 | valid tokens/s: 74595.4 | physical tokens/s: 75567.1 | loss tokens/s: 56376.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.287075E-01 | lm-global-supervised-tokens: 1.267953E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.595356E-02 | lm-token-buffer-mib: 2.365737E+02 | global_load_balancing_loss: 1.002040E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11204.581 | params rms: 0.060669 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:08:30.197991 | finish at 2026-10-17 19:02:21 + [2026-10-13 01:57:39] iteration 574/ 2384 | consumed samples: 18368 | elapsed time per iteration (ms): 228190.7 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.47% | FLOPs estimator: workload-aware | logical tokens/s: 73522.6 | valid tokens/s: 73522.6 | physical tokens/s: 73955.8 | loss tokens/s: 53748.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.414078E-01 | lm-global-supervised-tokens: 1.226494E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.210841E-02 | lm-token-buffer-mib: 2.333975E+02 | global_load_balancing_loss: 1.001313E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11204.549 | params rms: 0.060669 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 18:43:45.239596 | finish at 2026-10-17 20:41:24 + [2026-10-13 02:01:30] iteration 575/ 2384 | consumed samples: 18400 | elapsed time per iteration (ms): 231515.5 | throughput per GPU (TFLOP/s/GPU): 19.0 | MFU 6.10% | FLOPs estimator: workload-aware | logical tokens/s: 72466.8 | valid tokens/s: 72466.8 | physical tokens/s: 73569.9 | loss tokens/s: 54781.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.750300E-01 | lm-global-supervised-tokens: 1.268284E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 6.965396E-01 | lm-token-buffer-mib: 2.346661E+02 | global_load_balancing_loss: 1.000179E+00 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11204.518 | params rms: 0.060669 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:20:11.584911 | finish at 2026-10-17 22:21:42 + [2026-10-13 02:05:20] iteration 576/ 2384 | consumed samples: 18432 | elapsed time per iteration (ms): 229729.8 | throughput per GPU (TFLOP/s/GPU): 23.4 | MFU 7.51% | FLOPs estimator: workload-aware | logical tokens/s: 73030.1 | valid tokens/s: 73030.1 | physical tokens/s: 73883.0 | loss tokens/s: 54629.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.301012E-01 | lm-global-supervised-tokens: 1.255012E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.299895E-02 | lm-token-buffer-mib: 2.367236E+02 | global_load_balancing_loss: 1.000232E+00 | loss scale: 1.0 | grad norm: 0.102 | params norm: 11204.488 | params rms: 0.060669 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:22:31.467598 | finish at 2026-10-17 21:27:51 + [2026-10-13 02:09:01] iteration 577/ 2384 | consumed samples: 18464 | elapsed time per iteration (ms): 220638.1 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.83% | FLOPs estimator: workload-aware | logical tokens/s: 76039.5 | valid tokens/s: 76039.5 | physical tokens/s: 76271.9 | loss tokens/s: 51213.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.130396E-01 | lm-global-supervised-tokens: 1.129963E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.294178E-02 | lm-token-buffer-mib: 2.326840E+02 | global_load_balancing_loss: 9.984412E-01 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11204.454 | params rms: 0.060668 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:44:53.003453 | finish at 2026-10-17 16:53:54 + [2026-10-13 02:12:42] iteration 578/ 2384 | consumed samples: 18496 | elapsed time per iteration (ms): 221625.1 | throughput per GPU (TFLOP/s/GPU): 23.5 | MFU 7.53% | FLOPs estimator: workload-aware | logical tokens/s: 75700.8 | valid tokens/s: 75700.8 | physical tokens/s: 76048.3 | loss tokens/s: 58819.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.114124E-01 | lm-global-supervised-tokens: 1.303591E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.501013E-02 | lm-token-buffer-mib: 2.328669E+02 | global_load_balancing_loss: 9.993094E-01 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11204.416 | params rms: 0.060668 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:10:54.869272 | finish at 2026-10-17 17:23:37 + [2026-10-13 02:16:22] iteration 579/ 2384 | consumed samples: 18528 | elapsed time per iteration (ms): 220050.3 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.02% | FLOPs estimator: workload-aware | logical tokens/s: 76242.4 | valid tokens/s: 76242.3 | physical tokens/s: 77013.1 | loss tokens/s: 56171.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.686311E-01 | lm-global-supervised-tokens: 1.236045E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.693961E-02 | lm-token-buffer-mib: 2.337024E+02 | global_load_balancing_loss: 9.994152E-01 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11204.383 | params rms: 0.060668 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:19:50.833462 | finish at 2026-10-17 16:36:13 + [2026-10-13 02:20:25] iteration 580/ 2384 | consumed samples: 18560 | elapsed time per iteration (ms): 242777.6 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.08% | FLOPs estimator: workload-aware | logical tokens/s: 69105.1 | valid tokens/s: 69105.1 | physical tokens/s: 69726.4 | loss tokens/s: 51313.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.829905E-01 | lm-global-supervised-tokens: 1.245771E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.198496E-02 | lm-token-buffer-mib: 2.335604E+02 | global_load_balancing_loss: 9.995284E-01 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11204.350 | params rms: 0.060668 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:39:30.871780 | finish at 2026-10-18 03:59:56 + [2026-10-13 02:24:08] iteration 581/ 2384 | consumed samples: 18592 | elapsed time per iteration (ms): 222530.2 | throughput per GPU (TFLOP/s/GPU): 26.0 | MFU 8.33% | FLOPs estimator: workload-aware | logical tokens/s: 75392.7 | valid tokens/s: 75392.7 | physical tokens/s: 75694.1 | loss tokens/s: 54344.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.689075E-01 | lm-global-supervised-tokens: 1.209338E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.999877E-02 | lm-token-buffer-mib: 2.328524E+02 | global_load_balancing_loss: 1.003002E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11204.312 | params rms: 0.060668 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:27:02.013369 | finish at 2026-10-17 17:51:10 + [2026-10-13 02:27:52] iteration 582/ 2384 | consumed samples: 18624 | elapsed time per iteration (ms): 224968.9 | throughput per GPU (TFLOP/s/GPU): 24.6 | MFU 7.88% | FLOPs estimator: workload-aware | logical tokens/s: 74575.6 | valid tokens/s: 74575.6 | physical tokens/s: 75362.6 | loss tokens/s: 56865.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.675546E-01 | lm-global-supervised-tokens: 1.279287E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.737363E-02 | lm-token-buffer-mib: 2.332477E+02 | global_load_balancing_loss: 1.005943E+00 | loss scale: 1.0 | grad norm: 0.178 | params norm: 11204.273 | params rms: 0.060667 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:36:34.042804 | finish at 2026-10-17 19:04:27 + [2026-10-13 02:31:32] iteration 583/ 2384 | consumed samples: 18656 | elapsed time per iteration (ms): 219219.6 | throughput per GPU (TFLOP/s/GPU): 18.4 | MFU 5.91% | FLOPs estimator: workload-aware | logical tokens/s: 76531.5 | valid tokens/s: 76531.5 | physical tokens/s: 77237.2 | loss tokens/s: 61531.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.776514E-01 | lm-global-supervised-tokens: 1.348886E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.795881E-02 | lm-token-buffer-mib: 2.332269E+02 | global_load_balancing_loss: 1.001071E+00 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11204.234 | params rms: 0.060667 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:40:14.470333 | finish at 2026-10-17 16:11:46 + [2026-10-13 02:35:15] iteration 584/ 2384 | consumed samples: 18688 | elapsed time per iteration (ms): 223597.8 | throughput per GPU (TFLOP/s/GPU): 24.6 | MFU 7.89% | FLOPs estimator: workload-aware | logical tokens/s: 75032.9 | valid tokens/s: 75032.9 | physical tokens/s: 75636.1 | loss tokens/s: 60449.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.016011E-01 | lm-global-supervised-tokens: 1.351630E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.671587E-02 | lm-token-buffer-mib: 2.339234E+02 | global_load_balancing_loss: 1.001232E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11204.196 | params rms: 0.060667 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:47:56.090240 | finish at 2026-10-17 18:23:11 + [2026-10-13 02:38:59] iteration 585/ 2384 | consumed samples: 18720 | elapsed time per iteration (ms): 223943.4 | throughput per GPU (TFLOP/s/GPU): 24.1 | MFU 7.71% | FLOPs estimator: workload-aware | logical tokens/s: 74917.2 | valid tokens/s: 74917.2 | physical tokens/s: 75288.5 | loss tokens/s: 61258.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.135029E-01 | lm-global-supervised-tokens: 1.371844E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.386728E-02 | lm-token-buffer-mib: 2.332994E+02 | global_load_balancing_loss: 1.001688E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11204.154 | params rms: 0.060667 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:54:34.102658 | finish at 2026-10-17 18:33:33 + [2026-10-13 02:42:42] iteration 586/ 2384 | consumed samples: 18752 | elapsed time per iteration (ms): 222964.0 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.02% | FLOPs estimator: workload-aware | logical tokens/s: 75246.1 | valid tokens/s: 75246.1 | physical tokens/s: 75976.5 | loss tokens/s: 59102.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.903259E-01 | lm-global-supervised-tokens: 1.317774E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.457065E-02 | lm-token-buffer-mib: 2.342523E+02 | global_load_balancing_loss: 9.988787E-01 | loss scale: 1.0 | grad norm: 0.190 | params norm: 11204.110 | params rms: 0.060667 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:21:29.356854 | finish at 2026-10-17 18:04:12 + [2026-10-13 02:46:26] iteration 587/ 2384 | consumed samples: 18784 | elapsed time per iteration (ms): 223710.8 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.91% | FLOPs estimator: workload-aware | logical tokens/s: 74995.0 | valid tokens/s: 74995.0 | physical tokens/s: 75740.6 | loss tokens/s: 56832.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.841326E-01 | lm-global-supervised-tokens: 1.271402E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.470576E-02 | lm-token-buffer-mib: 2.344933E+02 | global_load_balancing_loss: 9.974823E-01 | loss scale: 1.0 | grad norm: 0.168 | params norm: 11204.066 | params rms: 0.060666 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:40:08.280915 | finish at 2026-10-17 18:26:34 + [2026-10-13 02:50:08] iteration 588/ 2384 | consumed samples: 18816 | elapsed time per iteration (ms): 222287.4 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.56% | FLOPs estimator: workload-aware | logical tokens/s: 75475.3 | valid tokens/s: 75475.3 | physical tokens/s: 75885.0 | loss tokens/s: 55080.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.617960E-01 | lm-global-supervised-tokens: 1.224364E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.562313E-02 | lm-token-buffer-mib: 2.325891E+02 | global_load_balancing_loss: 9.995146E-01 | loss scale: 1.0 | grad norm: 0.146 | params norm: 11204.023 | params rms: 0.060666 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:53:48.116888 | finish at 2026-10-17 17:43:56 + [2026-10-13 02:53:51] iteration 589/ 2384 | consumed samples: 18848 | elapsed time per iteration (ms): 222282.1 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.62% | FLOPs estimator: workload-aware | logical tokens/s: 75477.0 | valid tokens/s: 75477.0 | physical tokens/s: 76743.4 | loss tokens/s: 59017.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.932295E-01 | lm-global-supervised-tokens: 1.311859E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.335755E-02 | lm-token-buffer-mib: 2.350905E+02 | global_load_balancing_loss: 9.994560E-01 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11203.980 | params rms: 0.060666 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:49:56.365581 | finish at 2026-10-17 17:43:47 + [2026-10-13 02:57:56] iteration 590/ 2384 | consumed samples: 18880 | elapsed time per iteration (ms): 245472.3 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.14% | FLOPs estimator: workload-aware | logical tokens/s: 68346.4 | valid tokens/s: 68346.4 | physical tokens/s: 68459.6 | loss tokens/s: 46005.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.072661E-01 | lm-global-supervised-tokens: 1.129302E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.681165E-02 | lm-token-buffer-mib: 2.322645E+02 | global_load_balancing_loss: 1.002416E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11203.939 | params rms: 0.060666 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:19:37.302017 | finish at 2026-10-18 05:17:33 + [2026-10-13 03:02:30] iteration 591/ 2384 | consumed samples: 18912 | elapsed time per iteration (ms): 273905.0 | throughput per GPU (TFLOP/s/GPU): 18.6 | MFU 5.95% | FLOPs estimator: workload-aware | logical tokens/s: 61251.9 | valid tokens/s: 61251.9 | physical tokens/s: 62486.4 | loss tokens/s: 49101.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.880160E-01 | lm-global-supervised-tokens: 1.344909E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.947806E-02 | lm-token-buffer-mib: 2.400619E+02 | global_load_balancing_loss: 1.001467E+00 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11203.893 | params rms: 0.060665 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:25:11.709835 | finish at 2026-10-18 19:27:42 + [2026-10-13 03:06:10] iteration 592/ 2384 | consumed samples: 18944 | elapsed time per iteration (ms): 220251.2 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.27% | FLOPs estimator: workload-aware | logical tokens/s: 76173.1 | valid tokens/s: 76173.1 | physical tokens/s: 76729.9 | loss tokens/s: 54946.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.292004E-01 | lm-global-supervised-tokens: 1.210199E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.277787E-02 | lm-token-buffer-mib: 2.335594E+02 | global_load_balancing_loss: 1.001464E+00 | loss scale: 1.0 | grad norm: 0.105 | params norm: 11203.848 | params rms: 0.060665 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:38:10.101196 | finish at 2026-10-17 16:44:20 + [2026-10-13 03:09:55] iteration 593/ 2384 | consumed samples: 18976 | elapsed time per iteration (ms): 224380.4 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.04% | FLOPs estimator: workload-aware | logical tokens/s: 74771.3 | valid tokens/s: 74771.3 | physical tokens/s: 75276.5 | loss tokens/s: 58843.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.758725E-01 | lm-global-supervised-tokens: 1.320325E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.637481E-02 | lm-token-buffer-mib: 2.335656E+02 | global_load_balancing_loss: 1.000558E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11203.798 | params rms: 0.060665 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:37:45.227549 | finish at 2026-10-17 18:47:40 + [2026-10-13 03:13:36] iteration 594/ 2384 | consumed samples: 19008 | elapsed time per iteration (ms): 221666.8 | throughput per GPU (TFLOP/s/GPU): 24.0 | MFU 7.68% | FLOPs estimator: workload-aware | logical tokens/s: 75686.4 | valid tokens/s: 75686.4 | physical tokens/s: 76138.2 | loss tokens/s: 59130.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.056802E-01 | lm-global-supervised-tokens: 1.310729E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.839070E-02 | lm-token-buffer-mib: 2.340498E+02 | global_load_balancing_loss: 9.984189E-01 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11203.745 | params rms: 0.060665 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:13:03.591244 | finish at 2026-10-17 17:26:40 + [2026-10-13 03:17:22] iteration 595/ 2384 | consumed samples: 19040 | elapsed time per iteration (ms): 225544.8 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.12% | FLOPs estimator: workload-aware | logical tokens/s: 74385.2 | valid tokens/s: 74385.2 | physical tokens/s: 75130.2 | loss tokens/s: 57692.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.343420E-01 | lm-global-supervised-tokens: 1.301219E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.909078E-02 | lm-token-buffer-mib: 2.336053E+02 | global_load_balancing_loss: 1.001197E+00 | loss scale: 1.0 | grad norm: 0.191 | params norm: 11203.691 | params rms: 0.060664 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:04:59.654102 | finish at 2026-10-17 19:22:21 + [2026-10-13 03:21:05] iteration 596/ 2384 | consumed samples: 19072 | elapsed time per iteration (ms): 223039.6 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.88% | FLOPs estimator: workload-aware | logical tokens/s: 75220.7 | valid tokens/s: 75220.7 | physical tokens/s: 76696.8 | loss tokens/s: 53736.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.598952E-01 | lm-global-supervised-tokens: 1.198533E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.499308E-02 | lm-token-buffer-mib: 2.367990E+02 | global_load_balancing_loss: 9.992596E-01 | loss scale: 1.0 | grad norm: 0.357 | params norm: 11203.636 | params rms: 0.060664 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:46:34.811860 | finish at 2026-10-17 18:07:40 + [2026-10-13 03:24:46] iteration 597/ 2384 | consumed samples: 19104 | elapsed time per iteration (ms): 221421.4 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.63% | FLOPs estimator: workload-aware | logical tokens/s: 75770.5 | valid tokens/s: 75770.5 | physical tokens/s: 76358.4 | loss tokens/s: 56762.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.370320E-01 | lm-global-supervised-tokens: 1.256836E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.674327E-02 | lm-token-buffer-mib: 2.352552E+02 | global_load_balancing_loss: 9.996377E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11203.583 | params rms: 0.060664 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:54:40.088791 | finish at 2026-10-17 17:19:26 + [2026-10-13 03:28:30] iteration 598/ 2384 | consumed samples: 19136 | elapsed time per iteration (ms): 223934.8 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.46% | FLOPs estimator: workload-aware | logical tokens/s: 74920.0 | valid tokens/s: 74920.0 | physical tokens/s: 75664.9 | loss tokens/s: 58275.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.657025E-01 | lm-global-supervised-tokens: 1.305001E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.443296E-02 | lm-token-buffer-mib: 2.340999E+02 | global_load_balancing_loss: 1.001133E+00 | loss scale: 1.0 | grad norm: 0.226 | params norm: 11203.531 | params rms: 0.060663 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:05:47.614382 | finish at 2026-10-17 18:34:18 + [2026-10-13 03:32:12] iteration 599/ 2384 | consumed samples: 19168 | elapsed time per iteration (ms): 222238.2 | throughput per GPU (TFLOP/s/GPU): 23.7 | MFU 7.59% | FLOPs estimator: workload-aware | logical tokens/s: 75491.8 | valid tokens/s: 75491.8 | physical tokens/s: 76110.3 | loss tokens/s: 55493.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.745400E-01 | lm-global-supervised-tokens: 1.233285E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.258851E-02 | lm-token-buffer-mib: 2.336561E+02 | global_load_balancing_loss: 9.995465E-01 | loss scale: 1.0 | grad norm: 0.110 | params norm: 11203.477 | params rms: 0.060663 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:11:35.147331 | finish at 2026-10-17 17:43:48 + [2026-10-13 03:36:16] iteration 600/ 2384 | consumed samples: 19200 | elapsed time per iteration (ms): 243753.8 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.51% | FLOPs estimator: workload-aware | logical tokens/s: 68828.4 | valid tokens/s: 68828.4 | physical tokens/s: 69459.2 | loss tokens/s: 52201.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.055326E-01 | lm-global-supervised-tokens: 1.272436E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 6.371637E-02 | lm-token-buffer-mib: 2.344358E+02 | global_load_balancing_loss: 1.002605E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11203.425 | params rms: 0.060663 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:47:36.818605 | finish at 2026-10-18 04:23:53 +times across ranks (ms): + save-checkpoint: + rank 0: 25671.15 + rank 1: 25671.18 + rank 2: 25671.21 + rank 3: 25671.18 + rank 4: 25671.18 + rank 5: 25671.20 + rank 6: 25671.21 + rank 7: 25671.18 + rank 8: 25670.93 + rank 9: 25670.98 + rank 10: 25670.97 + rank 11: 25670.94 + rank 12: 25670.98 + rank 13: 25670.93 + rank 14: 25670.93 + rank 15: 25670.93 + rank 16: 25669.48 + rank 17: 25669.48 + rank 18: 25669.50 + rank 19: 25669.51 + rank 20: 25669.46 + rank 21: 25669.45 + rank 22: 25669.49 + rank 23: 25669.46 + rank 24: 25670.44 + rank 25: 25670.47 + rank 26: 25670.40 + rank 27: 25670.38 + rank 28: 25670.15 + rank 29: 25670.45 + rank 30: 25670.47 + rank 31: 25670.46 + rank 32: 25670.60 + rank 33: 25670.51 + rank 34: 25670.66 + rank 35: 25670.56 + rank 36: 25670.51 + rank 37: 25670.72 + rank 38: 25670.56 + rank 39: 25670.49 + rank 40: 25670.84 + rank 41: 25670.87 + rank 42: 25670.88 + rank 43: 25670.86 + rank 44: 25670.84 + rank 45: 25670.84 + rank 46: 25670.83 + rank 47: 25670.83 + rank 48: 25670.79 + rank 49: 25670.76 + rank 50: 25670.91 + rank 51: 25670.79 + rank 52: 25670.80 + rank 53: 25670.78 + rank 54: 25670.87 + rank 55: 25670.79 + rank 56: 25669.99 + rank 57: 25670.02 + rank 58: 25670.02 + rank 59: 25670.01 + rank 60: 25670.02 + rank 61: 25669.98 + rank 62: 25669.99 + rank 63: 25669.99 + [2026-10-13 03:40:21] iteration 601/ 2384 | consumed samples: 19232 | elapsed time per iteration (ms): 219154.7 | throughput per GPU (TFLOP/s/GPU): 26.6 | MFU 8.52% | FLOPs estimator: workload-aware | logical tokens/s: 76554.1 | valid tokens/s: 76554.0 | physical tokens/s: 77122.8 | loss tokens/s: 64341.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.662224E-01 | lm-global-supervised-tokens: 1.410065E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.338111E-01 | lm-token-buffer-mib: 2.341162E+02 | global_load_balancing_loss: 1.005542E+00 | loss scale: 1.0 | grad norm: 0.191 | params norm: 11203.373 | params rms: 0.060663 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:32:32.783412 | finish at 2026-10-17 16:12:54 + [2026-10-13 03:44:05] iteration 602/ 2384 | consumed samples: 19264 | elapsed time per iteration (ms): 224383.1 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.77% | FLOPs estimator: workload-aware | logical tokens/s: 74770.3 | valid tokens/s: 74770.3 | physical tokens/s: 75496.0 | loss tokens/s: 57369.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.388222E-01 | lm-global-supervised-tokens: 1.287275E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.781816E-02 | lm-token-buffer-mib: 2.339387E+02 | global_load_balancing_loss: 1.001450E+00 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11203.320 | params rms: 0.060662 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:04:10.760734 | finish at 2026-10-17 18:48:16 + [2026-10-13 03:47:51] iteration 603/ 2384 | consumed samples: 19296 | elapsed time per iteration (ms): 225821.8 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.93% | FLOPs estimator: workload-aware | logical tokens/s: 74293.9 | valid tokens/s: 74293.9 | physical tokens/s: 75106.0 | loss tokens/s: 52301.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.620785E-01 | lm-global-supervised-tokens: 1.181090E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.609265E-02 | lm-token-buffer-mib: 2.350935E+02 | global_load_balancing_loss: 1.000572E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11203.266 | params rms: 0.060662 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:43:08.677593 | finish at 2026-10-17 19:31:00 + [2026-10-13 03:51:34] iteration 604/ 2384 | consumed samples: 19328 | elapsed time per iteration (ms): 222388.1 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.88% | FLOPs estimator: workload-aware | logical tokens/s: 75441.1 | valid tokens/s: 75441.1 | physical tokens/s: 76359.5 | loss tokens/s: 59699.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.665571E-01 | lm-global-supervised-tokens: 1.327639E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.273328E-02 | lm-token-buffer-mib: 2.354129E+02 | global_load_balancing_loss: 9.996791E-01 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11203.210 | params rms: 0.060662 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:57:30.786009 | finish at 2026-10-17 17:49:04 + [2026-10-13 03:55:15] iteration 605/ 2384 | consumed samples: 19360 | elapsed time per iteration (ms): 221574.6 | throughput per GPU (TFLOP/s/GPU): 23.2 | MFU 7.44% | FLOPs estimator: workload-aware | logical tokens/s: 75718.1 | valid tokens/s: 75718.0 | physical tokens/s: 75934.8 | loss tokens/s: 53132.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.238971E-01 | lm-global-supervised-tokens: 1.177290E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.545798E-02 | lm-token-buffer-mib: 2.326534E+02 | global_load_balancing_loss: 9.996078E-01 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11203.157 | params rms: 0.060661 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:29:41.128537 | finish at 2026-10-17 17:24:56 + [2026-10-13 03:58:56] iteration 606/ 2384 | consumed samples: 19392 | elapsed time per iteration (ms): 220836.5 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.89% | FLOPs estimator: workload-aware | logical tokens/s: 75971.2 | valid tokens/s: 75971.2 | physical tokens/s: 76649.4 | loss tokens/s: 58233.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.301130E-01 | lm-global-supervised-tokens: 1.286007E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.280066E-02 | lm-token-buffer-mib: 2.334947E+02 | global_load_balancing_loss: 9.993919E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11203.105 | params rms: 0.060661 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:04:07.264657 | finish at 2026-10-17 17:03:03 + [2026-10-13 04:02:38] iteration 607/ 2384 | consumed samples: 19424 | elapsed time per iteration (ms): 221593.2 | throughput per GPU (TFLOP/s/GPU): 25.2 | MFU 8.07% | FLOPs estimator: workload-aware | logical tokens/s: 75711.6 | valid tokens/s: 75711.6 | physical tokens/s: 76537.5 | loss tokens/s: 58154.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.392829E-01 | lm-global-supervised-tokens: 1.288656E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.410982E-02 | lm-token-buffer-mib: 2.353925E+02 | global_load_balancing_loss: 9.999895E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11203.053 | params rms: 0.060661 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:22:51.041354 | finish at 2026-10-17 17:25:29 + [2026-10-13 04:06:19] iteration 608/ 2384 | consumed samples: 19456 | elapsed time per iteration (ms): 221670.5 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.81% | FLOPs estimator: workload-aware | logical tokens/s: 75685.2 | valid tokens/s: 75685.2 | physical tokens/s: 76354.1 | loss tokens/s: 59363.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.534832E-01 | lm-global-supervised-tokens: 1.315914E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.679457E-02 | lm-token-buffer-mib: 2.345078E+02 | global_load_balancing_loss: 1.000132E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11203.003 | params rms: 0.060661 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:21:26.788593 | finish at 2026-10-17 17:27:46 + [2026-10-13 04:10:03] iteration 609/ 2384 | consumed samples: 19488 | elapsed time per iteration (ms): 223507.2 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.93% | FLOPs estimator: workload-aware | logical tokens/s: 75063.4 | valid tokens/s: 75063.4 | physical tokens/s: 76149.8 | loss tokens/s: 58674.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.320322E-01 | lm-global-supervised-tokens: 1.311425E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.790782E-02 | lm-token-buffer-mib: 2.366143E+02 | global_load_balancing_loss: 1.000745E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11202.954 | params rms: 0.060660 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:12:05.326556 | finish at 2026-10-17 18:22:08 + [2026-10-13 04:14:09] iteration 610/ 2384 | consumed samples: 19520 | elapsed time per iteration (ms): 245948.7 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.57% | FLOPs estimator: workload-aware | logical tokens/s: 68214.3 | valid tokens/s: 68214.3 | physical tokens/s: 68659.3 | loss tokens/s: 54130.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.395326E-01 | lm-global-supervised-tokens: 1.331319E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.516113E-02 | lm-token-buffer-mib: 2.337636E+02 | global_load_balancing_loss: 1.000923E+00 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11202.904 | params rms: 0.060660 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:11:52.910814 | finish at 2026-10-18 05:26:02 + [2026-10-13 04:17:53] iteration 611/ 2384 | consumed samples: 19552 | elapsed time per iteration (ms): 224461.0 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.78% | FLOPs estimator: workload-aware | logical tokens/s: 74744.1 | valid tokens/s: 74744.1 | physical tokens/s: 76110.5 | loss tokens/s: 59343.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.661605E-01 | lm-global-supervised-tokens: 1.332024E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.696688E-02 | lm-token-buffer-mib: 2.340556E+02 | global_load_balancing_loss: 1.001353E+00 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11202.857 | params rms: 0.060660 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:32:49.385911 | finish at 2026-10-17 18:50:43 + [2026-10-13 04:21:37] iteration 612/ 2384 | consumed samples: 19584 | elapsed time per iteration (ms): 223877.4 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.11% | FLOPs estimator: workload-aware | logical tokens/s: 74939.3 | valid tokens/s: 74939.3 | physical tokens/s: 75637.1 | loss tokens/s: 57278.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.734189E-01 | lm-global-supervised-tokens: 1.282344E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.342267E-02 | lm-token-buffer-mib: 2.338633E+02 | global_load_balancing_loss: 1.002135E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11202.811 | params rms: 0.060660 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:11:50.791529 | finish at 2026-10-17 18:33:28 + [2026-10-13 04:25:19] iteration 613/ 2384 | consumed samples: 19616 | elapsed time per iteration (ms): 221893.9 | throughput per GPU (TFLOP/s/GPU): 25.8 | MFU 8.28% | FLOPs estimator: workload-aware | logical tokens/s: 75609.0 | valid tokens/s: 75609.0 | physical tokens/s: 75910.3 | loss tokens/s: 56413.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.338480E-01 | lm-global-supervised-tokens: 1.251788E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.999231E-02 | lm-token-buffer-mib: 2.331407E+02 | global_load_balancing_loss: 1.004156E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11202.761 | params rms: 0.060659 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:09:34.136445 | finish at 2026-10-17 17:34:53 + [2026-10-13 04:29:01] iteration 614/ 2384 | consumed samples: 19648 | elapsed time per iteration (ms): 222040.1 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.56% | FLOPs estimator: workload-aware | logical tokens/s: 75559.4 | valid tokens/s: 75559.4 | physical tokens/s: 76248.3 | loss tokens/s: 57300.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.420022E-01 | lm-global-supervised-tokens: 1.272302E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.335292E-02 | lm-token-buffer-mib: 2.344160E+02 | global_load_balancing_loss: 1.000625E+00 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11202.712 | params rms: 0.060659 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:10:11.000383 | finish at 2026-10-17 17:39:12 + [2026-10-13 04:32:45] iteration 615/ 2384 | consumed samples: 19680 | elapsed time per iteration (ms): 223680.7 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.78% | FLOPs estimator: workload-aware | logical tokens/s: 75005.1 | valid tokens/s: 75005.1 | physical tokens/s: 75785.8 | loss tokens/s: 57235.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.740895E-01 | lm-global-supervised-tokens: 1.280246E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.208005E-02 | lm-token-buffer-mib: 2.334775E+02 | global_load_balancing_loss: 9.996907E-01 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11202.668 | params rms: 0.060659 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:54:51.096414 | finish at 2026-10-17 18:27:36 + [2026-10-13 04:36:28] iteration 616/ 2384 | consumed samples: 19712 | elapsed time per iteration (ms): 223020.8 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.63% | FLOPs estimator: workload-aware | logical tokens/s: 75227.1 | valid tokens/s: 75227.1 | physical tokens/s: 75505.5 | loss tokens/s: 52763.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.175991E-01 | lm-global-supervised-tokens: 1.176729E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.587383E-02 | lm-token-buffer-mib: 2.327155E+02 | global_load_balancing_loss: 9.962447E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11202.625 | params rms: 0.060659 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:31:40.799892 | finish at 2026-10-17 18:08:09 + [2026-10-13 04:40:10] iteration 617/ 2384 | consumed samples: 19744 | elapsed time per iteration (ms): 222441.6 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.84% | FLOPs estimator: workload-aware | logical tokens/s: 75423.0 | valid tokens/s: 75423.0 | physical tokens/s: 76260.9 | loss tokens/s: 56848.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.027050E-01 | lm-global-supervised-tokens: 1.264550E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.112820E-02 | lm-token-buffer-mib: 2.327810E+02 | global_load_balancing_loss: 9.980180E-01 | loss scale: 1.0 | grad norm: 0.096 | params norm: 11202.581 | params rms: 0.060658 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:10:54.245420 | finish at 2026-10-17 17:51:04 + [2026-10-13 04:43:50] iteration 618/ 2384 | consumed samples: 19776 | elapsed time per iteration (ms): 219489.9 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.14% | FLOPs estimator: workload-aware | logical tokens/s: 76437.3 | valid tokens/s: 76437.3 | physical tokens/s: 76937.1 | loss tokens/s: 56614.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.134213E-01 | lm-global-supervised-tokens: 1.242628E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.345174E-02 | lm-token-buffer-mib: 2.330936E+02 | global_load_balancing_loss: 1.000447E+00 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11202.536 | params rms: 0.060658 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:40:19.145914 | finish at 2026-10-17 16:24:09 + [2026-10-13 04:47:29] iteration 619/ 2384 | consumed samples: 19808 | elapsed time per iteration (ms): 219653.1 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.95% | FLOPs estimator: workload-aware | logical tokens/s: 76380.3 | valid tokens/s: 76380.3 | physical tokens/s: 77283.5 | loss tokens/s: 56444.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.453431E-01 | lm-global-supervised-tokens: 1.239822E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.813039E-02 | lm-token-buffer-mib: 2.328650E+02 | global_load_balancing_loss: 1.002038E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11202.489 | params rms: 0.060658 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:41:27.673973 | finish at 2026-10-17 16:28:57 + [2026-10-13 04:51:33] iteration 620/ 2384 | consumed samples: 19840 | elapsed time per iteration (ms): 243439.0 | throughput per GPU (TFLOP/s/GPU): 19.6 | MFU 6.29% | FLOPs estimator: workload-aware | logical tokens/s: 68917.3 | valid tokens/s: 68917.3 | physical tokens/s: 69438.1 | loss tokens/s: 51761.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.711902E-01 | lm-global-supervised-tokens: 1.260078E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.528551E-02 | lm-token-buffer-mib: 2.334406E+02 | global_load_balancing_loss: 1.001708E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11202.441 | params rms: 0.060658 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:17:06.392444 | finish at 2026-10-18 04:08:39 + [2026-10-13 04:55:15] iteration 621/ 2384 | consumed samples: 19872 | elapsed time per iteration (ms): 222335.8 | throughput per GPU (TFLOP/s/GPU): 25.1 | MFU 8.03% | FLOPs estimator: workload-aware | logical tokens/s: 75458.7 | valid tokens/s: 75458.7 | physical tokens/s: 76529.7 | loss tokens/s: 55656.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.108674E-01 | lm-global-supervised-tokens: 1.237445E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.966786E-02 | lm-token-buffer-mib: 2.349687E+02 | global_load_balancing_loss: 1.002823E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11202.397 | params rms: 0.060657 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:52:58.085476 | finish at 2026-10-17 17:48:13 + [2026-10-13 04:58:57] iteration 622/ 2384 | consumed samples: 19904 | elapsed time per iteration (ms): 221439.3 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.90% | FLOPs estimator: workload-aware | logical tokens/s: 75764.3 | valid tokens/s: 75764.3 | physical tokens/s: 76531.7 | loss tokens/s: 57185.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.990704E-01 | lm-global-supervised-tokens: 1.266311E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.572749E-02 | lm-token-buffer-mib: 2.335959E+02 | global_load_balancing_loss: 1.001548E+00 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11202.354 | params rms: 0.060657 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:22:56.039145 | finish at 2026-10-17 17:21:53 + [2026-10-13 05:02:39] iteration 623/ 2384 | consumed samples: 19936 | elapsed time per iteration (ms): 221992.0 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.91% | FLOPs estimator: workload-aware | logical tokens/s: 75575.5 | valid tokens/s: 75575.5 | physical tokens/s: 76106.5 | loss tokens/s: 54223.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.844882E-01 | lm-global-supervised-tokens: 1.203726E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.737830E-02 | lm-token-buffer-mib: 2.334669E+02 | global_load_balancing_loss: 1.000390E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11202.309 | params rms: 0.060657 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:35:27.973060 | finish at 2026-10-17 17:38:06 + [2026-10-13 05:06:21] iteration 624/ 2384 | consumed samples: 19968 | elapsed time per iteration (ms): 222044.0 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.48% | FLOPs estimator: workload-aware | logical tokens/s: 75558.0 | valid tokens/s: 75558.0 | physical tokens/s: 75865.3 | loss tokens/s: 56156.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.201777E-01 | lm-global-supervised-tokens: 1.246927E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.275910E-02 | lm-token-buffer-mib: 2.325933E+02 | global_load_balancing_loss: 1.001470E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11202.263 | params rms: 0.060657 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:33:17.479706 | finish at 2026-10-17 17:39:38 + [2026-10-13 05:10:04] iteration 625/ 2384 | consumed samples: 20000 | elapsed time per iteration (ms): 223358.8 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.40% | FLOPs estimator: workload-aware | logical tokens/s: 75113.1 | valid tokens/s: 75113.1 | physical tokens/s: 75846.2 | loss tokens/s: 58608.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.082367E-01 | lm-global-supervised-tokens: 1.309063E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.422829E-02 | lm-token-buffer-mib: 2.347194E+02 | global_load_balancing_loss: 1.000788E+00 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11202.215 | params rms: 0.060656 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:08:08.196604 | finish at 2026-10-17 18:18:12 + [2026-10-13 05:13:50] iteration 626/ 2384 | consumed samples: 20032 | elapsed time per iteration (ms): 225733.4 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.46% | FLOPs estimator: workload-aware | logical tokens/s: 74323.1 | valid tokens/s: 74323.1 | physical tokens/s: 75086.7 | loss tokens/s: 56723.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.743099E-01 | lm-global-supervised-tokens: 1.280436E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.471409E-02 | lm-token-buffer-mib: 2.337211E+02 | global_load_balancing_loss: 9.989275E-01 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11202.168 | params rms: 0.060656 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:13:59.385707 | finish at 2026-10-17 19:27:49 + [2026-10-13 05:17:33] iteration 627/ 2384 | consumed samples: 20064 | elapsed time per iteration (ms): 223241.1 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.84% | FLOPs estimator: workload-aware | logical tokens/s: 75152.8 | valid tokens/s: 75152.8 | physical tokens/s: 75678.4 | loss tokens/s: 55021.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.416176E-01 | lm-global-supervised-tokens: 1.228297E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.081004E-02 | lm-token-buffer-mib: 2.346298E+02 | global_load_balancing_loss: 1.000740E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11202.122 | params rms: 0.060656 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:57:14.668123 | finish at 2026-10-17 18:14:48 + [2026-10-13 05:21:12] iteration 628/ 2384 | consumed samples: 20096 | elapsed time per iteration (ms): 219262.3 | throughput per GPU (TFLOP/s/GPU): 18.2 | MFU 5.84% | FLOPs estimator: workload-aware | logical tokens/s: 76516.6 | valid tokens/s: 76516.6 | physical tokens/s: 77174.6 | loss tokens/s: 58356.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.854762E-01 | lm-global-supervised-tokens: 1.279542E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.430358E-02 | lm-token-buffer-mib: 2.331735E+02 | global_load_balancing_loss: 1.001534E+00 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11202.077 | params rms: 0.060656 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:57:04.523466 | finish at 2026-10-17 16:18:17 + [2026-10-13 05:24:55] iteration 629/ 2384 | consumed samples: 20128 | elapsed time per iteration (ms): 222708.1 | throughput per GPU (TFLOP/s/GPU): 24.7 | MFU 7.91% | FLOPs estimator: workload-aware | logical tokens/s: 75332.6 | valid tokens/s: 75332.6 | physical tokens/s: 75734.8 | loss tokens/s: 55364.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.023188E-01 | lm-global-supervised-tokens: 1.233003E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.477985E-02 | lm-token-buffer-mib: 2.327313E+02 | global_load_balancing_loss: 1.000183E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11202.032 | params rms: 0.060655 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:34:12.639488 | finish at 2026-10-17 17:59:08 + [2026-10-13 05:29:01] iteration 630/ 2384 | consumed samples: 20160 | elapsed time per iteration (ms): 246434.0 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.76% | FLOPs estimator: workload-aware | logical tokens/s: 68079.9 | valid tokens/s: 68079.9 | physical tokens/s: 68791.8 | loss tokens/s: 50427.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.231961E-01 | lm-global-supervised-tokens: 1.242708E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.795591E-02 | lm-token-buffer-mib: 2.334043E+02 | global_load_balancing_loss: 1.000531E+00 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11201.991 | params rms: 0.060655 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:04:05.270318 | finish at 2026-10-18 05:33:07 + [2026-10-13 05:32:45] iteration 631/ 2384 | consumed samples: 20192 | elapsed time per iteration (ms): 223474.7 | throughput per GPU (TFLOP/s/GPU): 24.7 | MFU 7.91% | FLOPs estimator: workload-aware | logical tokens/s: 75074.2 | valid tokens/s: 75074.2 | physical tokens/s: 76031.7 | loss tokens/s: 59334.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.527460E-01 | lm-global-supervised-tokens: 1.325982E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.591830E-02 | lm-token-buffer-mib: 2.354978E+02 | global_load_balancing_loss: 9.997386E-01 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11201.951 | params rms: 0.060655 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:49:11.095139 | finish at 2026-10-17 18:21:56 + [2026-10-13 05:36:26] iteration 632/ 2384 | consumed samples: 20224 | elapsed time per iteration (ms): 221611.5 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.07% | FLOPs estimator: workload-aware | logical tokens/s: 75705.3 | valid tokens/s: 75705.3 | physical tokens/s: 76369.2 | loss tokens/s: 54888.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.816506E-01 | lm-global-supervised-tokens: 1.216393E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.065815E-02 | lm-token-buffer-mib: 2.342267E+02 | global_load_balancing_loss: 1.000625E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11201.912 | params rms: 0.060655 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:51:03.285805 | finish at 2026-10-17 17:27:30 + [2026-10-13 05:40:08] iteration 633/ 2384 | consumed samples: 20256 | elapsed time per iteration (ms): 221271.6 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.25% | FLOPs estimator: workload-aware | logical tokens/s: 75821.6 | valid tokens/s: 75821.6 | physical tokens/s: 76551.0 | loss tokens/s: 56159.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.747613E-01 | lm-global-supervised-tokens: 1.242645E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.423437E-02 | lm-token-buffer-mib: 2.333101E+02 | global_load_balancing_loss: 9.995530E-01 | loss scale: 1.0 | grad norm: 0.123 | params norm: 11201.874 | params rms: 0.060655 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:37:26.602924 | finish at 2026-10-17 17:17:34 + [2026-10-13 05:44:01] iteration 634/ 2384 | consumed samples: 20288 | elapsed time per iteration (ms): 233295.0 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.56% | FLOPs estimator: workload-aware | logical tokens/s: 71914.0 | valid tokens/s: 71914.0 | physical tokens/s: 73075.4 | loss tokens/s: 51947.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.511941E-01 | lm-global-supervised-tokens: 1.211915E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.257440E-02 | lm-token-buffer-mib: 2.389834E+02 | global_load_balancing_loss: 9.997566E-01 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11201.833 | params rms: 0.060654 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:24:26.254306 | finish at 2026-10-17 23:08:27 + [2026-10-13 05:47:43] iteration 635/ 2384 | consumed samples: 20320 | elapsed time per iteration (ms): 221623.4 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.04% | FLOPs estimator: workload-aware | logical tokens/s: 75701.3 | valid tokens/s: 75701.3 | physical tokens/s: 76365.7 | loss tokens/s: 58815.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.381399E-01 | lm-global-supervised-tokens: 1.303480E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.214845E-02 | lm-token-buffer-mib: 2.325991E+02 | global_load_balancing_loss: 1.000428E+00 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11201.791 | params rms: 0.060654 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:40:19.296946 | finish at 2026-10-17 17:28:02 + [2026-10-13 05:51:29] iteration 636/ 2384 | consumed samples: 20352 | elapsed time per iteration (ms): 225996.1 | throughput per GPU (TFLOP/s/GPU): 18.6 | MFU 5.97% | FLOPs estimator: workload-aware | logical tokens/s: 74236.7 | valid tokens/s: 74236.7 | physical tokens/s: 75736.2 | loss tokens/s: 56442.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.027473E-01 | lm-global-supervised-tokens: 1.275580E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.752362E-02 | lm-token-buffer-mib: 2.384713E+02 | global_load_balancing_loss: 1.000821E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11201.753 | params rms: 0.060654 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:44:01.205215 | finish at 2026-10-17 19:35:30 + [2026-10-13 05:55:09] iteration 637/ 2384 | consumed samples: 20384 | elapsed time per iteration (ms): 220526.6 | throughput per GPU (TFLOP/s/GPU): 20.2 | MFU 6.48% | FLOPs estimator: workload-aware | logical tokens/s: 76077.9 | valid tokens/s: 76077.9 | physical tokens/s: 76630.5 | loss tokens/s: 57676.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.405331E-01 | lm-global-supervised-tokens: 1.271910E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.095434E-02 | lm-token-buffer-mib: 2.338465E+02 | global_load_balancing_loss: 1.000030E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11201.717 | params rms: 0.060654 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:00:59.927929 | finish at 2026-10-17 16:56:09 + [2026-10-13 05:58:51] iteration 638/ 2384 | consumed samples: 20416 | elapsed time per iteration (ms): 221801.6 | throughput per GPU (TFLOP/s/GPU): 19.7 | MFU 6.31% | FLOPs estimator: workload-aware | logical tokens/s: 75640.6 | valid tokens/s: 75640.6 | physical tokens/s: 76701.3 | loss tokens/s: 61399.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.265190E-01 | lm-global-supervised-tokens: 1.361853E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.812198E-02 | lm-token-buffer-mib: 2.360113E+02 | global_load_balancing_loss: 1.000648E+00 | loss scale: 1.0 | grad norm: 0.108 | params norm: 11201.678 | params rms: 0.060653 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:34:25.646015 | finish at 2026-10-17 17:33:17 + [2026-10-13 06:02:36] iteration 639/ 2384 | consumed samples: 20448 | elapsed time per iteration (ms): 224756.9 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.64% | FLOPs estimator: workload-aware | logical tokens/s: 74645.7 | valid tokens/s: 74645.7 | physical tokens/s: 75553.0 | loss tokens/s: 52332.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.058490E-01 | lm-global-supervised-tokens: 1.176200E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.507937E-02 | lm-token-buffer-mib: 2.345776E+02 | global_load_balancing_loss: 1.000437E+00 | loss scale: 1.0 | grad norm: 0.110 | params norm: 11201.638 | params rms: 0.060653 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:56:40.850952 | finish at 2026-10-17 18:59:17 + [2026-10-13 06:06:40] iteration 640/ 2384 | consumed samples: 20480 | elapsed time per iteration (ms): 244017.2 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.23% | FLOPs estimator: workload-aware | logical tokens/s: 68754.1 | valid tokens/s: 68754.1 | physical tokens/s: 69091.0 | loss tokens/s: 49115.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.298720E-01 | lm-global-supervised-tokens: 1.198509E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.399739E-02 | lm-token-buffer-mib: 2.333215E+02 | global_load_balancing_loss: 1.001011E+00 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11201.598 | params rms: 0.060653 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:12:46.017162 | finish at 2026-10-18 04:19:26 +times across ranks (ms): + save-checkpoint: + rank 0: 24729.34 + rank 1: 24729.37 + rank 2: 24729.37 + rank 3: 24729.38 + rank 4: 24729.41 + rank 5: 24729.39 + rank 6: 24729.41 + rank 7: 24729.39 + rank 8: 24730.63 + rank 9: 24730.60 + rank 10: 24730.65 + rank 11: 24730.61 + rank 12: 24730.61 + rank 13: 24730.61 + rank 14: 24730.61 + rank 15: 24730.61 + rank 16: 24730.94 + rank 17: 24730.92 + rank 18: 24730.92 + rank 19: 24730.92 + rank 20: 24730.88 + rank 21: 24730.88 + rank 22: 24730.90 + rank 23: 24730.88 + rank 24: 24731.61 + rank 25: 24731.61 + rank 26: 24731.61 + rank 27: 24731.62 + rank 28: 24731.59 + rank 29: 24731.36 + rank 30: 24731.58 + rank 31: 24731.63 + rank 32: 24730.51 + rank 33: 24730.53 + rank 34: 24730.49 + rank 35: 24730.51 + rank 36: 24730.51 + rank 37: 24730.52 + rank 38: 24730.51 + rank 39: 24730.51 + rank 40: 24730.68 + rank 41: 24730.65 + rank 42: 24730.65 + rank 43: 24730.66 + rank 44: 24730.69 + rank 45: 24730.71 + rank 46: 24730.63 + rank 47: 24730.63 + rank 48: 24730.81 + rank 49: 24731.07 + rank 50: 24730.86 + rank 51: 24730.82 + rank 52: 24730.89 + rank 53: 24730.94 + rank 54: 24730.91 + rank 55: 24730.81 + rank 56: 24730.58 + rank 57: 24730.61 + rank 58: 24730.61 + rank 59: 24730.60 + rank 60: 24730.67 + rank 61: 24730.57 + rank 62: 24730.58 + rank 63: 24730.58 + [2026-10-13 06:10:50] iteration 641/ 2384 | consumed samples: 20512 | elapsed time per iteration (ms): 225309.0 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.51% | FLOPs estimator: workload-aware | logical tokens/s: 74463.0 | valid tokens/s: 74463.0 | physical tokens/s: 75494.3 | loss tokens/s: 56938.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.254026E-01 | lm-global-supervised-tokens: 1.282870E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.796557E-02 | lm-token-buffer-mib: 2.375279E+02 | global_load_balancing_loss: 1.001028E+00 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11201.559 | params rms: 0.060653 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:05:13.533419 | finish at 2026-10-17 19:16:03 + [2026-10-13 06:14:38] iteration 642/ 2384 | consumed samples: 20544 | elapsed time per iteration (ms): 228321.1 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.06% | FLOPs estimator: workload-aware | logical tokens/s: 73480.8 | valid tokens/s: 73480.8 | physical tokens/s: 73972.4 | loss tokens/s: 57088.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.304783E-01 | lm-global-supervised-tokens: 1.303459E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.394258E-02 | lm-token-buffer-mib: 2.334234E+02 | global_load_balancing_loss: 1.000296E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11201.521 | params rms: 0.060653 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:28:55.377376 | finish at 2026-10-17 20:43:33 + [2026-10-13 06:18:18] iteration 643/ 2384 | consumed samples: 20576 | elapsed time per iteration (ms): 219960.7 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.71% | FLOPs estimator: workload-aware | logical tokens/s: 76273.6 | valid tokens/s: 76273.6 | physical tokens/s: 77024.9 | loss tokens/s: 54667.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.797281E-01 | lm-global-supervised-tokens: 1.202477E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.099049E-02 | lm-token-buffer-mib: 2.343063E+02 | global_load_balancing_loss: 9.998002E-01 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11201.483 | params rms: 0.060652 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:22:31.655967 | finish at 2026-10-17 16:40:50 + [2026-10-13 06:22:03] iteration 644/ 2384 | consumed samples: 20608 | elapsed time per iteration (ms): 225255.6 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.88% | FLOPs estimator: workload-aware | logical tokens/s: 74480.7 | valid tokens/s: 74480.7 | physical tokens/s: 75867.0 | loss tokens/s: 54125.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.501955E-01 | lm-global-supervised-tokens: 1.219197E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.639647E-02 | lm-token-buffer-mib: 2.370901E+02 | global_load_balancing_loss: 1.000727E+00 | loss scale: 1.0 | grad norm: 0.113 | params norm: 11201.444 | params rms: 0.060652 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:52:24.725289 | finish at 2026-10-17 19:14:28 + [2026-10-13 06:25:47] iteration 645/ 2384 | consumed samples: 20640 | elapsed time per iteration (ms): 223594.6 | throughput per GPU (TFLOP/s/GPU): 23.5 | MFU 7.53% | FLOPs estimator: workload-aware | logical tokens/s: 75034.1 | valid tokens/s: 75034.1 | physical tokens/s: 76010.1 | loss tokens/s: 55973.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.736365E-01 | lm-global-supervised-tokens: 1.251526E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.423732E-02 | lm-token-buffer-mib: 2.342921E+02 | global_load_balancing_loss: 1.002832E+00 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11201.406 | params rms: 0.060652 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:00:31.051625 | finish at 2026-10-17 18:26:18 + [2026-10-13 06:29:27] iteration 646/ 2384 | consumed samples: 20672 | elapsed time per iteration (ms): 220081.1 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.45% | FLOPs estimator: workload-aware | logical tokens/s: 76231.9 | valid tokens/s: 76231.9 | physical tokens/s: 77250.9 | loss tokens/s: 57895.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.033999E-01 | lm-global-supervised-tokens: 1.274176E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.576268E-02 | lm-token-buffer-mib: 2.327570E+02 | global_load_balancing_loss: 1.000086E+00 | loss scale: 1.0 | grad norm: 0.147 | params norm: 11201.373 | params rms: 0.060652 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:15:00.943076 | finish at 2026-10-17 16:44:28 + [2026-10-13 06:33:06] iteration 647/ 2384 | consumed samples: 20704 | elapsed time per iteration (ms): 219111.2 | throughput per GPU (TFLOP/s/GPU): 18.7 | MFU 6.00% | FLOPs estimator: workload-aware | logical tokens/s: 76569.4 | valid tokens/s: 76569.4 | physical tokens/s: 77165.0 | loss tokens/s: 55125.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.742067E-01 | lm-global-supervised-tokens: 1.207851E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.515361E-02 | lm-token-buffer-mib: 2.331495E+02 | global_load_balancing_loss: 9.993759E-01 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11201.339 | params rms: 0.060652 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:43:16.078777 | finish at 2026-10-17 16:16:22 + [2026-10-13 06:36:50] iteration 648/ 2384 | consumed samples: 20736 | elapsed time per iteration (ms): 223472.4 | throughput per GPU (TFLOP/s/GPU): 24.0 | MFU 7.70% | FLOPs estimator: workload-aware | logical tokens/s: 75075.0 | valid tokens/s: 75075.0 | physical tokens/s: 75986.1 | loss tokens/s: 56628.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.401956E-01 | lm-global-supervised-tokens: 1.265492E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.348166E-02 | lm-token-buffer-mib: 2.344633E+02 | global_load_balancing_loss: 9.986839E-01 | loss scale: 1.0 | grad norm: 0.119 | params norm: 11201.300 | params rms: 0.060651 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:45:48.090866 | finish at 2026-10-17 18:22:38 + [2026-10-13 06:40:35] iteration 649/ 2384 | consumed samples: 20768 | elapsed time per iteration (ms): 225760.1 | throughput per GPU (TFLOP/s/GPU): 24.6 | MFU 7.89% | FLOPs estimator: workload-aware | logical tokens/s: 74314.2 | valid tokens/s: 74314.2 | physical tokens/s: 74981.4 | loss tokens/s: 49600.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.537059E-01 | lm-global-supervised-tokens: 1.119770E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.601054E-02 | lm-token-buffer-mib: 2.347198E+02 | global_load_balancing_loss: 9.999738E-01 | loss scale: 1.0 | grad norm: 0.106 | params norm: 11201.259 | params rms: 0.060651 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:48:13.800607 | finish at 2026-10-17 19:28:49 + [2026-10-13 06:44:37] iteration 650/ 2384 | consumed samples: 20800 | elapsed time per iteration (ms): 241545.4 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.88% | FLOPs estimator: workload-aware | logical tokens/s: 69457.5 | valid tokens/s: 69457.5 | physical tokens/s: 69970.8 | loss tokens/s: 52643.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.067364E-01 | lm-global-supervised-tokens: 1.271589E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.986692E-02 | lm-token-buffer-mib: 2.332457E+02 | global_load_balancing_loss: 9.999787E-01 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11201.217 | params rms: 0.060651 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:20:39.807358 | finish at 2026-10-18 03:05:17 + [2026-10-13 06:48:15] iteration 651/ 2384 | consumed samples: 20832 | elapsed time per iteration (ms): 218098.8 | throughput per GPU (TFLOP/s/GPU): 24.2 | MFU 7.77% | FLOPs estimator: workload-aware | logical tokens/s: 76924.7 | valid tokens/s: 76924.7 | physical tokens/s: 77328.0 | loss tokens/s: 58779.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.364879E-01 | lm-global-supervised-tokens: 1.281966E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.597040E-02 | lm-token-buffer-mib: 2.334258E+02 | global_load_balancing_loss: 1.002494E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11201.177 | params rms: 0.060651 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:59:25.168655 | finish at 2026-10-17 15:47:40 + [2026-10-13 06:52:00] iteration 652/ 2384 | consumed samples: 20864 | elapsed time per iteration (ms): 224886.1 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.10% | FLOPs estimator: workload-aware | logical tokens/s: 74603.1 | valid tokens/s: 74603.1 | physical tokens/s: 75360.7 | loss tokens/s: 54647.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.460218E-01 | lm-global-supervised-tokens: 1.228952E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.741330E-02 | lm-token-buffer-mib: 2.335074E+02 | global_load_balancing_loss: 1.002096E+00 | loss scale: 1.0 | grad norm: 0.114 | params norm: 11201.134 | params rms: 0.060650 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:11:42.776498 | finish at 2026-10-17 19:03:43 + [2026-10-13 06:55:45] iteration 653/ 2384 | consumed samples: 20896 | elapsed time per iteration (ms): 225289.7 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.64% | FLOPs estimator: workload-aware | logical tokens/s: 74469.3 | valid tokens/s: 74469.3 | physical tokens/s: 75072.9 | loss tokens/s: 55893.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.421369E-01 | lm-global-supervised-tokens: 1.259227E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.707870E-02 | lm-token-buffer-mib: 2.341161E+02 | global_load_balancing_loss: 1.001837E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11201.095 | params rms: 0.060650 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:19:36.386563 | finish at 2026-10-17 19:15:22 + [2026-10-13 06:59:27] iteration 654/ 2384 | consumed samples: 20928 | elapsed time per iteration (ms): 222078.2 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.80% | FLOPs estimator: workload-aware | logical tokens/s: 75546.4 | valid tokens/s: 75546.4 | physical tokens/s: 76627.4 | loss tokens/s: 54447.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.186051E-01 | lm-global-supervised-tokens: 1.209159E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.558021E-02 | lm-token-buffer-mib: 2.348950E+02 | global_load_balancing_loss: 1.000748E+00 | loss scale: 1.0 | grad norm: 0.123 | params norm: 11201.058 | params rms: 0.060650 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:43:15.249879 | finish at 2026-10-17 17:42:43 + [2026-10-13 07:03:09] iteration 655/ 2384 | consumed samples: 20960 | elapsed time per iteration (ms): 221805.6 | throughput per GPU (TFLOP/s/GPU): 23.5 | MFU 7.53% | FLOPs estimator: workload-aware | logical tokens/s: 75639.1 | valid tokens/s: 75639.1 | physical tokens/s: 76072.1 | loss tokens/s: 56368.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.556603E-01 | lm-global-supervised-tokens: 1.250286E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.624434E-02 | lm-token-buffer-mib: 2.329162E+02 | global_load_balancing_loss: 9.991020E-01 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11201.016 | params rms: 0.060650 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:31:41.900826 | finish at 2026-10-17 17:34:51 + [2026-10-13 07:06:58] iteration 656/ 2384 | consumed samples: 20992 | elapsed time per iteration (ms): 229074.8 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.76% | FLOPs estimator: workload-aware | logical tokens/s: 73238.9 | valid tokens/s: 73238.9 | physical tokens/s: 74753.8 | loss tokens/s: 58967.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.983770E-01 | lm-global-supervised-tokens: 1.350802E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.692005E-02 | lm-token-buffer-mib: 2.386452E+02 | global_load_balancing_loss: 9.992747E-01 | loss scale: 1.0 | grad norm: 0.400 | params norm: 11200.972 | params rms: 0.060650 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:57:21.284500 | finish at 2026-10-17 21:04:19 + [2026-10-13 07:10:40] iteration 657/ 2384 | consumed samples: 21024 | elapsed time per iteration (ms): 221376.9 | throughput per GPU (TFLOP/s/GPU): 19.6 | MFU 6.29% | FLOPs estimator: workload-aware | logical tokens/s: 75785.6 | valid tokens/s: 75785.6 | physical tokens/s: 76253.5 | loss tokens/s: 57014.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.288342E-01 | lm-global-supervised-tokens: 1.262160E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.457123E-02 | lm-token-buffer-mib: 2.332248E+02 | global_load_balancing_loss: 9.994736E-01 | loss scale: 1.0 | grad norm: 0.186 | params norm: 11200.929 | params rms: 0.060649 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:11:57.932167 | finish at 2026-10-17 17:22:37 + [2026-10-13 07:14:22] iteration 658/ 2384 | consumed samples: 21056 | elapsed time per iteration (ms): 222369.9 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.33% | FLOPs estimator: workload-aware | logical tokens/s: 75447.0 | valid tokens/s: 75447.0 | physical tokens/s: 76275.1 | loss tokens/s: 55668.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.515880E-01 | lm-global-supervised-tokens: 1.237894E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.701279E-02 | lm-token-buffer-mib: 2.348268E+02 | global_load_balancing_loss: 1.001500E+00 | loss scale: 1.0 | grad norm: 0.316 | params norm: 11200.889 | params rms: 0.060649 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:36:50.419397 | finish at 2026-10-17 17:51:12 + [2026-10-13 07:18:07] iteration 659/ 2384 | consumed samples: 21088 | elapsed time per iteration (ms): 224876.0 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.04% | FLOPs estimator: workload-aware | logical tokens/s: 74606.4 | valid tokens/s: 74606.4 | physical tokens/s: 75242.9 | loss tokens/s: 55761.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.831228E-01 | lm-global-supervised-tokens: 1.253946E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.809410E-02 | lm-token-buffer-mib: 2.356993E+02 | global_load_balancing_loss: 1.001690E+00 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11200.850 | params rms: 0.060649 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:45:11.074787 | finish at 2026-10-17 19:03:18 + [2026-10-13 07:22:14] iteration 660/ 2384 | consumed samples: 21120 | elapsed time per iteration (ms): 247633.9 | throughput per GPU (TFLOP/s/GPU): 25.8 | MFU 8.26% | FLOPs estimator: workload-aware | logical tokens/s: 67750.0 | valid tokens/s: 67750.0 | physical tokens/s: 68331.8 | loss tokens/s: 52468.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.009838E-01 | lm-global-supervised-tokens: 1.299299E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.213079E-02 | lm-token-buffer-mib: 2.353724E+02 | global_load_balancing_loss: 1.003753E+00 | loss scale: 1.0 | grad norm: 0.108 | params norm: 11200.810 | params rms: 0.060649 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:35:20.822923 | finish at 2026-10-18 05:57:35 + [2026-10-13 07:25:58] iteration 661/ 2384 | consumed samples: 21152 | elapsed time per iteration (ms): 223820.1 | throughput per GPU (TFLOP/s/GPU): 24.6 | MFU 7.89% | FLOPs estimator: workload-aware | logical tokens/s: 74958.4 | valid tokens/s: 74958.4 | physical tokens/s: 75821.2 | loss tokens/s: 53416.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.211795E-01 | lm-global-supervised-tokens: 1.195564E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.466135E-02 | lm-token-buffer-mib: 2.350478E+02 | global_load_balancing_loss: 9.990667E-01 | loss scale: 1.0 | grad norm: 0.312 | params norm: 11200.774 | params rms: 0.060649 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:07:22.081304 | finish at 2026-10-17 18:33:20 + [2026-10-13 07:29:40] iteration 662/ 2384 | consumed samples: 21184 | elapsed time per iteration (ms): 222232.7 | throughput per GPU (TFLOP/s/GPU): 23.4 | MFU 7.49% | FLOPs estimator: workload-aware | logical tokens/s: 75493.7 | valid tokens/s: 75493.7 | physical tokens/s: 76135.0 | loss tokens/s: 58946.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.713885E-01 | lm-global-supervised-tokens: 1.309994E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.886215E-02 | lm-token-buffer-mib: 2.333672E+02 | global_load_balancing_loss: 9.988997E-01 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11200.739 | params rms: 0.060648 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:18:04.760498 | finish at 2026-10-17 17:47:45 + [2026-10-13 07:33:22] iteration 663/ 2384 | consumed samples: 21216 | elapsed time per iteration (ms): 221681.0 | throughput per GPU (TFLOP/s/GPU): 18.6 | MFU 5.96% | FLOPs estimator: workload-aware | logical tokens/s: 75681.7 | valid tokens/s: 75681.7 | physical tokens/s: 76821.9 | loss tokens/s: 58085.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.269855E-01 | lm-global-supervised-tokens: 1.287636E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.278296E-02 | lm-token-buffer-mib: 2.360391E+02 | global_load_balancing_loss: 9.996835E-01 | loss scale: 1.0 | grad norm: 0.170 | params norm: 11200.704 | params rms: 0.060648 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:58:33.039560 | finish at 2026-10-17 17:31:55 + [2026-10-13 07:37:03] iteration 664/ 2384 | consumed samples: 21248 | elapsed time per iteration (ms): 220832.8 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.61% | FLOPs estimator: workload-aware | logical tokens/s: 75972.3 | valid tokens/s: 75972.3 | physical tokens/s: 76453.0 | loss tokens/s: 58887.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.348532E-01 | lm-global-supervised-tokens: 1.300424E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.630024E-02 | lm-token-buffer-mib: 2.337663E+02 | global_load_balancing_loss: 9.986721E-01 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11200.670 | params rms: 0.060648 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:30:32.409286 | finish at 2026-10-17 17:07:35 + [2026-10-13 07:40:43] iteration 665/ 2384 | consumed samples: 21280 | elapsed time per iteration (ms): 219738.6 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.20% | FLOPs estimator: workload-aware | logical tokens/s: 76350.6 | valid tokens/s: 76350.6 | physical tokens/s: 76851.5 | loss tokens/s: 57347.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.592218E-01 | lm-global-supervised-tokens: 1.260147E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.673127E-02 | lm-token-buffer-mib: 2.340905E+02 | global_load_balancing_loss: 1.000639E+00 | loss scale: 1.0 | grad norm: 0.164 | params norm: 11200.634 | params rms: 0.060648 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:55:30.624328 | finish at 2026-10-17 16:36:13 + [2026-10-13 07:44:24] iteration 666/ 2384 | consumed samples: 21312 | elapsed time per iteration (ms): 221495.4 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.55% | FLOPs estimator: workload-aware | logical tokens/s: 75745.0 | valid tokens/s: 75745.0 | physical tokens/s: 76022.3 | loss tokens/s: 55367.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.874415E-01 | lm-global-supervised-tokens: 1.226365E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.201132E-02 | lm-token-buffer-mib: 2.328575E+02 | global_load_balancing_loss: 1.001720E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11200.592 | params rms: 0.060648 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:42:09.115959 | finish at 2026-10-17 17:26:33 + [2026-10-13 07:48:09] iteration 667/ 2384 | consumed samples: 21344 | elapsed time per iteration (ms): 224558.7 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.58% | FLOPs estimator: workload-aware | logical tokens/s: 74711.8 | valid tokens/s: 74711.8 | physical tokens/s: 75348.3 | loss tokens/s: 54443.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.882483E-01 | lm-global-supervised-tokens: 1.222579E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.471407E-02 | lm-token-buffer-mib: 2.332906E+02 | global_load_balancing_loss: 1.004149E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11200.555 | params rms: 0.060647 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:06:07.370328 | finish at 2026-10-17 18:54:16 + [2026-10-13 07:51:52] iteration 668/ 2384 | consumed samples: 21376 | elapsed time per iteration (ms): 223122.2 | throughput per GPU (TFLOP/s/GPU): 23.5 | MFU 7.54% | FLOPs estimator: workload-aware | logical tokens/s: 75192.9 | valid tokens/s: 75192.9 | physical tokens/s: 76005.6 | loss tokens/s: 56264.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.135694E-01 | lm-global-supervised-tokens: 1.255377E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.573109E-02 | lm-token-buffer-mib: 2.350883E+02 | global_load_balancing_loss: 1.000986E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11200.522 | params rms: 0.060647 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:21:17.707904 | finish at 2026-10-17 18:13:10 + [2026-10-13 07:55:34] iteration 669/ 2384 | consumed samples: 21408 | elapsed time per iteration (ms): 222040.4 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.93% | FLOPs estimator: workload-aware | logical tokens/s: 75559.2 | valid tokens/s: 75559.2 | physical tokens/s: 75829.1 | loss tokens/s: 59681.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.825950E-01 | lm-global-supervised-tokens: 1.325170E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.505486E-02 | lm-token-buffer-mib: 2.330491E+02 | global_load_balancing_loss: 1.000938E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11200.491 | params rms: 0.060647 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:46:39.238617 | finish at 2026-10-17 17:42:13 + [2026-10-13 07:59:43] iteration 670/ 2384 | consumed samples: 21440 | elapsed time per iteration (ms): 249331.7 | throughput per GPU (TFLOP/s/GPU): 19.5 | MFU 6.24% | FLOPs estimator: workload-aware | logical tokens/s: 67288.7 | valid tokens/s: 67288.7 | physical tokens/s: 68470.3 | loss tokens/s: 54968.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.389323E-01 | lm-global-supervised-tokens: 1.370534E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.298723E-02 | lm-token-buffer-mib: 2.386116E+02 | global_load_balancing_loss: 1.001328E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11200.458 | params rms: 0.060647 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:42:34.464478 | finish at 2026-10-18 06:42:18 + [2026-10-13 08:03:24] iteration 671/ 2384 | consumed samples: 21472 | elapsed time per iteration (ms): 220815.5 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.77% | FLOPs estimator: workload-aware | logical tokens/s: 75978.3 | valid tokens/s: 75978.3 | physical tokens/s: 76597.0 | loss tokens/s: 52889.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.124175E-01 | lm-global-supervised-tokens: 1.167874E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.522256E-02 | lm-token-buffer-mib: 2.337910E+02 | global_load_balancing_loss: 9.996966E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11200.427 | params rms: 0.060647 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:04:16.868220 | finish at 2026-10-17 17:07:41 + [2026-10-13 08:07:02] iteration 672/ 2384 | consumed samples: 21504 | elapsed time per iteration (ms): 218123.8 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.30% | FLOPs estimator: workload-aware | logical tokens/s: 76916.0 | valid tokens/s: 76916.0 | physical tokens/s: 77402.2 | loss tokens/s: 57584.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.818440E-01 | lm-global-supervised-tokens: 1.256058E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.914552E-02 | lm-token-buffer-mib: 2.339250E+02 | global_load_balancing_loss: 1.000891E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11200.393 | params rms: 0.060646 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:43:47.864849 | finish at 2026-10-17 15:50:50 + [2026-10-13 08:10:45] iteration 673/ 2384 | consumed samples: 21536 | elapsed time per iteration (ms): 222403.8 | throughput per GPU (TFLOP/s/GPU): 23.8 | MFU 7.64% | FLOPs estimator: workload-aware | logical tokens/s: 75435.7 | valid tokens/s: 75435.7 | physical tokens/s: 75925.0 | loss tokens/s: 56169.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.021948E-01 | lm-global-supervised-tokens: 1.249229E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.340824E-02 | lm-token-buffer-mib: 2.328307E+02 | global_load_balancing_loss: 9.997603E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11200.358 | params rms: 0.060646 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:42:12.875710 | finish at 2026-10-17 17:52:57 + [2026-10-13 08:14:29] iteration 674/ 2384 | consumed samples: 21568 | elapsed time per iteration (ms): 223958.5 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.47% | FLOPs estimator: workload-aware | logical tokens/s: 74912.1 | valid tokens/s: 74912.1 | physical tokens/s: 75763.5 | loss tokens/s: 58575.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.927401E-01 | lm-global-supervised-tokens: 1.311836E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.192082E-02 | lm-token-buffer-mib: 2.344890E+02 | global_load_balancing_loss: 9.990363E-01 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11200.322 | params rms: 0.060646 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:22:49.057674 | finish at 2026-10-17 18:37:18 + [2026-10-13 08:18:12] iteration 675/ 2384 | consumed samples: 21600 | elapsed time per iteration (ms): 223730.9 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.93% | FLOPs estimator: workload-aware | logical tokens/s: 74988.2 | valid tokens/s: 74988.2 | physical tokens/s: 75717.0 | loss tokens/s: 55955.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.140324E-01 | lm-global-supervised-tokens: 1.251889E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.257861E-02 | lm-token-buffer-mib: 2.342758E+02 | global_load_balancing_loss: 9.996659E-01 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11200.286 | params rms: 0.060646 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:12:36.151375 | finish at 2026-10-17 18:30:48 + [2026-10-13 08:21:56] iteration 676/ 2384 | consumed samples: 21632 | elapsed time per iteration (ms): 224123.0 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.98% | FLOPs estimator: workload-aware | logical tokens/s: 74857.0 | valid tokens/s: 74857.0 | physical tokens/s: 75697.0 | loss tokens/s: 55413.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.839564E-01 | lm-global-supervised-tokens: 1.241936E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.632275E-02 | lm-token-buffer-mib: 2.366333E+02 | global_load_balancing_loss: 1.001561E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11200.250 | params rms: 0.060646 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:20:02.042711 | finish at 2026-10-17 18:41:58 + [2026-10-13 08:25:38] iteration 677/ 2384 | consumed samples: 21664 | elapsed time per iteration (ms): 222076.5 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.36% | FLOPs estimator: workload-aware | logical tokens/s: 75547.0 | valid tokens/s: 75547.0 | physical tokens/s: 76306.4 | loss tokens/s: 55355.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.847744E-01 | lm-global-supervised-tokens: 1.229325E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.419573E-02 | lm-token-buffer-mib: 2.339264E+02 | global_load_balancing_loss: 1.002855E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11200.215 | params rms: 0.060646 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:18:04.591501 | finish at 2026-10-17 17:43:43 + [2026-10-13 08:29:20] iteration 678/ 2384 | consumed samples: 21696 | elapsed time per iteration (ms): 221033.0 | throughput per GPU (TFLOP/s/GPU): 25.1 | MFU 8.03% | FLOPs estimator: workload-aware | logical tokens/s: 75903.5 | valid tokens/s: 75903.5 | physical tokens/s: 76193.2 | loss tokens/s: 52750.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.840025E-01 | lm-global-supervised-tokens: 1.165961E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.460272E-02 | lm-token-buffer-mib: 2.329827E+02 | global_load_balancing_loss: 1.001320E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11200.181 | params rms: 0.060645 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:44:42.306122 | finish at 2026-10-17 17:14:02 + [2026-10-13 08:33:03] iteration 679/ 2384 | consumed samples: 21728 | elapsed time per iteration (ms): 223027.3 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.36% | FLOPs estimator: workload-aware | logical tokens/s: 75224.8 | valid tokens/s: 75224.8 | physical tokens/s: 75959.9 | loss tokens/s: 60329.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.209194E-01 | lm-global-supervised-tokens: 1.345521E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.343098E-02 | lm-token-buffer-mib: 2.343170E+02 | global_load_balancing_loss: 1.001697E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11200.144 | params rms: 0.060645 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:37:41.502801 | finish at 2026-10-17 18:10:44 + [2026-10-13 08:37:06] iteration 680/ 2384 | consumed samples: 21760 | elapsed time per iteration (ms): 243597.4 | throughput per GPU (TFLOP/s/GPU): 19.9 | MFU 6.36% | FLOPs estimator: workload-aware | logical tokens/s: 68872.6 | valid tokens/s: 68872.6 | physical tokens/s: 69764.7 | loss tokens/s: 49688.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.316605E-01 | lm-global-supervised-tokens: 1.210393E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.382981E-02 | lm-token-buffer-mib: 2.344369E+02 | global_load_balancing_loss: 9.995846E-01 | loss scale: 1.0 | grad norm: 0.095 | params norm: 11200.109 | params rms: 0.060645 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:18:09.993484 | finish at 2026-10-18 03:55:16 +times across ranks (ms): + save-checkpoint: + rank 0: 24380.58 + rank 1: 24380.54 + rank 2: 24380.57 + rank 3: 24380.52 + rank 4: 24380.52 + rank 5: 24380.54 + rank 6: 24380.55 + rank 7: 24380.54 + rank 8: 24380.42 + rank 9: 24380.44 + rank 10: 24380.46 + rank 11: 24380.45 + rank 12: 24380.43 + rank 13: 24380.42 + rank 14: 24380.42 + rank 15: 24380.41 + rank 16: 24380.49 + rank 17: 24380.54 + rank 18: 24380.52 + rank 19: 24380.51 + rank 20: 24380.45 + rank 21: 24380.49 + rank 22: 24380.49 + rank 23: 24380.52 + rank 24: 24380.73 + rank 25: 24380.76 + rank 26: 24380.71 + rank 27: 24381.29 + rank 28: 24380.75 + rank 29: 24380.73 + rank 30: 24380.73 + rank 31: 24380.73 + rank 32: 24380.54 + rank 33: 24380.54 + rank 34: 24380.57 + rank 35: 24380.68 + rank 36: 24380.63 + rank 37: 24380.73 + rank 38: 24380.46 + rank 39: 24380.61 + rank 40: 24380.11 + rank 41: 24380.30 + rank 42: 24380.27 + rank 43: 24380.51 + rank 44: 24380.31 + rank 45: 24380.27 + rank 46: 24380.27 + rank 47: 24380.25 + rank 48: 24380.44 + rank 49: 24380.45 + rank 50: 24380.46 + rank 51: 24380.40 + rank 52: 24380.44 + rank 53: 24380.39 + rank 54: 24380.48 + rank 55: 24380.49 + rank 56: 24380.67 + rank 57: 24380.68 + rank 58: 24380.68 + rank 59: 24380.67 + rank 60: 24380.53 + rank 61: 24380.71 + rank 62: 24380.65 + rank 63: 24380.64 + [2026-10-13 08:41:14] iteration 681/ 2384 | consumed samples: 21792 | elapsed time per iteration (ms): 223222.1 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.76% | FLOPs estimator: workload-aware | logical tokens/s: 75159.1 | valid tokens/s: 75159.1 | physical tokens/s: 76648.5 | loss tokens/s: 61488.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.253965E-01 | lm-global-supervised-tokens: 1.372563E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.132287E-02 | lm-token-buffer-mib: 2.373254E+02 | global_load_balancing_loss: 9.996272E-01 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11200.073 | params rms: 0.060645 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:35:47.211159 | finish at 2026-10-17 18:17:01 + [2026-10-13 08:44:58] iteration 682/ 2384 | consumed samples: 21824 | elapsed time per iteration (ms): 224540.1 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.61% | FLOPs estimator: workload-aware | logical tokens/s: 74718.1 | valid tokens/s: 74718.1 | physical tokens/s: 75197.0 | loss tokens/s: 54539.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.197526E-01 | lm-global-supervised-tokens: 1.224633E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.803936E-02 | lm-token-buffer-mib: 2.333471E+02 | global_load_balancing_loss: 9.981944E-01 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11200.037 | params rms: 0.060645 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:09:27.269032 | finish at 2026-10-17 18:54:26 + [2026-10-13 08:48:39] iteration 683/ 2384 | consumed samples: 21856 | elapsed time per iteration (ms): 220920.4 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.56% | FLOPs estimator: workload-aware | logical tokens/s: 75942.2 | valid tokens/s: 75942.2 | physical tokens/s: 76400.9 | loss tokens/s: 56879.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.756373E-01 | lm-global-supervised-tokens: 1.256573E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.533832E-02 | lm-token-buffer-mib: 2.346188E+02 | global_load_balancing_loss: 9.981328E-01 | loss scale: 1.0 | grad norm: 0.205 | params norm: 11200.003 | params rms: 0.060644 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:23:05.609159 | finish at 2026-10-17 17:11:45 + [2026-10-13 08:52:24] iteration 684/ 2384 | consumed samples: 21888 | elapsed time per iteration (ms): 224623.4 | throughput per GPU (TFLOP/s/GPU): 24.5 | MFU 7.85% | FLOPs estimator: workload-aware | logical tokens/s: 74690.0 | valid tokens/s: 74690.0 | physical tokens/s: 75213.5 | loss tokens/s: 58970.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.478015E-01 | lm-global-supervised-tokens: 1.324607E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.712833E-02 | lm-token-buffer-mib: 2.339459E+02 | global_load_balancing_loss: 1.001039E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11199.964 | params rms: 0.060644 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:04:19.760094 | finish at 2026-10-17 18:56:44 + [2026-10-13 08:56:07] iteration 685/ 2384 | consumed samples: 21920 | elapsed time per iteration (ms): 223273.5 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.73% | FLOPs estimator: workload-aware | logical tokens/s: 75141.9 | valid tokens/s: 75141.9 | physical tokens/s: 76019.4 | loss tokens/s: 58590.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.737911E-01 | lm-global-supervised-tokens: 1.308173E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.356131E-02 | lm-token-buffer-mib: 2.348139E+02 | global_load_balancing_loss: 1.000999E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11199.927 | params rms: 0.060644 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:22:21.712898 | finish at 2026-10-17 18:18:29 + [2026-10-13 08:59:51] iteration 686/ 2384 | consumed samples: 21952 | elapsed time per iteration (ms): 224023.8 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.49% | FLOPs estimator: workload-aware | logical tokens/s: 74890.3 | valid tokens/s: 74890.3 | physical tokens/s: 75549.4 | loss tokens/s: 56831.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.555049E-01 | lm-global-supervised-tokens: 1.273170E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.402681E-02 | lm-token-buffer-mib: 2.333325E+02 | global_load_balancing_loss: 1.001444E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11199.894 | params rms: 0.060644 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:39:52.415058 | finish at 2026-10-17 18:39:44 + [2026-10-13 09:03:34] iteration 687/ 2384 | consumed samples: 21984 | elapsed time per iteration (ms): 222966.1 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.94% | FLOPs estimator: workload-aware | logical tokens/s: 75245.5 | valid tokens/s: 75245.5 | physical tokens/s: 76199.7 | loss tokens/s: 60584.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.132086E-01 | lm-global-supervised-tokens: 1.350822E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.748041E-02 | lm-token-buffer-mib: 2.363715E+02 | global_load_balancing_loss: 1.002371E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11199.863 | params rms: 0.060644 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:06:13.553390 | finish at 2026-10-17 18:09:48 + [2026-10-13 09:07:18] iteration 688/ 2384 | consumed samples: 22016 | elapsed time per iteration (ms): 223793.6 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.20% | FLOPs estimator: workload-aware | logical tokens/s: 74967.2 | valid tokens/s: 74967.2 | physical tokens/s: 75658.3 | loss tokens/s: 58925.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.126098E-01 | lm-global-supervised-tokens: 1.318711E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.263505E-02 | lm-token-buffer-mib: 2.354811E+02 | global_load_balancing_loss: 1.002985E+00 | loss scale: 1.0 | grad norm: 0.102 | params norm: 11199.832 | params rms: 0.060643 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:25:53.937653 | finish at 2026-10-17 18:33:12 + [2026-10-13 09:11:01] iteration 689/ 2384 | consumed samples: 22048 | elapsed time per iteration (ms): 223007.9 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.96% | FLOPs estimator: workload-aware | logical tokens/s: 75231.4 | valid tokens/s: 75231.4 | physical tokens/s: 75718.8 | loss tokens/s: 56362.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.965322E-01 | lm-global-supervised-tokens: 1.256933E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.285231E-02 | lm-token-buffer-mib: 2.335854E+02 | global_load_balancing_loss: 1.000750E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11199.799 | params rms: 0.060643 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:59:58.335943 | finish at 2026-10-17 18:10:59 + [2026-10-13 09:15:07] iteration 690/ 2384 | consumed samples: 22080 | elapsed time per iteration (ms): 245849.5 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.42% | FLOPs estimator: workload-aware | logical tokens/s: 68241.8 | valid tokens/s: 68241.8 | physical tokens/s: 68834.3 | loss tokens/s: 50552.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.833364E-01 | lm-global-supervised-tokens: 1.242838E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.673983E-02 | lm-token-buffer-mib: 2.326450E+02 | global_load_balancing_loss: 1.000684E+00 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11199.765 | params rms: 0.060643 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:41:09.067036 | finish at 2026-10-18 04:56:16 + [2026-10-13 09:18:51] iteration 691/ 2384 | consumed samples: 22112 | elapsed time per iteration (ms): 224372.2 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.29% | FLOPs estimator: workload-aware | logical tokens/s: 74774.0 | valid tokens/s: 74773.9 | physical tokens/s: 75753.6 | loss tokens/s: 59149.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.293000E-01 | lm-global-supervised-tokens: 1.327145E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.740435E-02 | lm-token-buffer-mib: 2.354397E+02 | global_load_balancing_loss: 9.978706E-01 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11199.731 | params rms: 0.060643 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:31:02.106366 | finish at 2026-10-17 18:49:53 + [2026-10-13 09:22:34] iteration 692/ 2384 | consumed samples: 22144 | elapsed time per iteration (ms): 223084.1 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.90% | FLOPs estimator: workload-aware | logical tokens/s: 75205.7 | valid tokens/s: 75205.7 | physical tokens/s: 75553.2 | loss tokens/s: 50953.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.404985E-01 | lm-global-supervised-tokens: 1.136688E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.780666E-02 | lm-token-buffer-mib: 2.330953E+02 | global_load_balancing_loss: 9.981178E-01 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11199.697 | params rms: 0.060643 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:50:58.333923 | finish at 2026-10-17 18:13:33 + [2026-10-13 09:26:19] iteration 693/ 2384 | consumed samples: 22176 | elapsed time per iteration (ms): 225146.6 | throughput per GPU (TFLOP/s/GPU): 19.5 | MFU 6.27% | FLOPs estimator: workload-aware | logical tokens/s: 74516.8 | valid tokens/s: 74516.8 | physical tokens/s: 75983.9 | loss tokens/s: 59619.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.782528E-01 | lm-global-supervised-tokens: 1.342323E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.538941E-02 | lm-token-buffer-mib: 2.384399E+02 | global_load_balancing_loss: 9.979254E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11199.661 | params rms: 0.060643 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:45:22.928432 | finish at 2026-10-17 19:11:42 + [2026-10-13 09:30:03] iteration 694/ 2384 | consumed samples: 22208 | elapsed time per iteration (ms): 223155.2 | throughput per GPU (TFLOP/s/GPU): 18.9 | MFU 6.04% | FLOPs estimator: workload-aware | logical tokens/s: 75181.7 | valid tokens/s: 75181.6 | physical tokens/s: 75760.9 | loss tokens/s: 59594.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.714948E-01 | lm-global-supervised-tokens: 1.329874E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.534382E-02 | lm-token-buffer-mib: 2.352128E+02 | global_load_balancing_loss: 1.001009E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11199.627 | params rms: 0.060642 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:45:32.248521 | finish at 2026-10-17 18:15:35 + [2026-10-13 09:33:45] iteration 695/ 2384 | consumed samples: 22240 | elapsed time per iteration (ms): 221990.9 | throughput per GPU (TFLOP/s/GPU): 24.0 | MFU 7.71% | FLOPs estimator: workload-aware | logical tokens/s: 75576.0 | valid tokens/s: 75576.0 | physical tokens/s: 76262.9 | loss tokens/s: 50839.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.871136E-01 | lm-global-supervised-tokens: 1.128600E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.142147E-02 | lm-token-buffer-mib: 2.324062E+02 | global_load_balancing_loss: 1.002451E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11199.596 | params rms: 0.060642 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:09:02.572985 | finish at 2026-10-17 17:42:47 + [2026-10-13 09:37:27] iteration 696/ 2384 | consumed samples: 22272 | elapsed time per iteration (ms): 222231.7 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.91% | FLOPs estimator: workload-aware | logical tokens/s: 75494.2 | valid tokens/s: 75494.2 | physical tokens/s: 76124.4 | loss tokens/s: 48184.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.616222E-01 | lm-global-supervised-tokens: 1.070815E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.390242E-02 | lm-token-buffer-mib: 2.337416E+02 | global_load_balancing_loss: 1.000780E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11199.565 | params rms: 0.060642 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:12:07.130835 | finish at 2026-10-17 17:49:34 + [2026-10-13 09:41:12] iteration 697/ 2384 | consumed samples: 22304 | elapsed time per iteration (ms): 225470.2 | throughput per GPU (TFLOP/s/GPU): 25.0 | MFU 8.02% | FLOPs estimator: workload-aware | logical tokens/s: 74409.7 | valid tokens/s: 74409.7 | physical tokens/s: 75252.7 | loss tokens/s: 50325.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.665372E-01 | lm-global-supervised-tokens: 1.134688E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.733702E-02 | lm-token-buffer-mib: 2.332947E+02 | global_load_balancing_loss: 1.000680E+00 | loss scale: 1.0 | grad norm: 0.124 | params norm: 11199.530 | params rms: 0.060642 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:39:28.248017 | finish at 2026-10-17 19:20:40 + [2026-10-13 09:44:58] iteration 698/ 2384 | consumed samples: 22336 | elapsed time per iteration (ms): 225520.8 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.56% | FLOPs estimator: workload-aware | logical tokens/s: 74393.2 | valid tokens/s: 74393.2 | physical tokens/s: 75098.7 | loss tokens/s: 54954.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.425840E-01 | lm-global-supervised-tokens: 1.239343E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.572190E-02 | lm-token-buffer-mib: 2.350925E+02 | global_load_balancing_loss: 1.000938E+00 | loss scale: 1.0 | grad norm: 0.137 | params norm: 11199.489 | params rms: 0.060642 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:37:08.106777 | finish at 2026-10-17 19:22:06 + [2026-10-13 09:48:39] iteration 699/ 2384 | consumed samples: 22368 | elapsed time per iteration (ms): 220955.0 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.39% | FLOPs estimator: workload-aware | logical tokens/s: 75930.5 | valid tokens/s: 75930.5 | physical tokens/s: 76427.5 | loss tokens/s: 58087.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.573003E-01 | lm-global-supervised-tokens: 1.283470E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.400715E-02 | lm-token-buffer-mib: 2.323359E+02 | global_load_balancing_loss: 9.992786E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11199.448 | params rms: 0.060641 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:25:09.182504 | finish at 2026-10-17 17:13:48 + [2026-10-13 09:52:42] iteration 700/ 2384 | consumed samples: 22400 | elapsed time per iteration (ms): 242818.6 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.64% | FLOPs estimator: workload-aware | logical tokens/s: 69093.5 | valid tokens/s: 69093.5 | physical tokens/s: 69703.5 | loss tokens/s: 52097.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.714126E-01 | lm-global-supervised-tokens: 1.265017E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.683836E-02 | lm-token-buffer-mib: 2.334579E+02 | global_load_balancing_loss: 9.995534E-01 | loss scale: 1.0 | grad norm: 0.184 | params norm: 11199.407 | params rms: 0.060641 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:35:06.584530 | finish at 2026-10-18 03:27:48 + [2026-10-13 09:56:25] iteration 701/ 2384 | consumed samples: 22432 | elapsed time per iteration (ms): 223007.3 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.10% | FLOPs estimator: workload-aware | logical tokens/s: 75231.6 | valid tokens/s: 75231.6 | physical tokens/s: 76074.0 | loss tokens/s: 55617.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.754493E-01 | lm-global-supervised-tokens: 1.240318E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.663826E-02 | lm-token-buffer-mib: 2.352243E+02 | global_load_balancing_loss: 1.001851E+00 | loss scale: 1.0 | grad norm: 0.191 | params norm: 11199.365 | params rms: 0.060641 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:15:21.288942 | finish at 2026-10-17 18:11:46 + [2026-10-13 10:00:07] iteration 702/ 2384 | consumed samples: 22464 | elapsed time per iteration (ms): 222740.4 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.63% | FLOPs estimator: workload-aware | logical tokens/s: 75321.8 | valid tokens/s: 75321.8 | physical tokens/s: 76459.4 | loss tokens/s: 56317.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.861268E-01 | lm-global-supervised-tokens: 1.254417E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.769516E-02 | lm-token-buffer-mib: 2.339440E+02 | global_load_balancing_loss: 1.001066E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11199.317 | params rms: 0.060641 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:04:09.310019 | finish at 2026-10-17 18:04:17 + [2026-10-13 10:03:51] iteration 703/ 2384 | consumed samples: 22496 | elapsed time per iteration (ms): 223917.6 | throughput per GPU (TFLOP/s/GPU): 19.9 | MFU 6.37% | FLOPs estimator: workload-aware | logical tokens/s: 74925.8 | valid tokens/s: 74925.8 | physical tokens/s: 75766.7 | loss tokens/s: 59217.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.878654E-01 | lm-global-supervised-tokens: 1.325976E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.736522E-02 | lm-token-buffer-mib: 2.347597E+02 | global_load_balancing_loss: 1.002565E+00 | loss scale: 1.0 | grad norm: 0.145 | params norm: 11199.267 | params rms: 0.060640 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:33:25.456203 | finish at 2026-10-17 18:37:17 + [2026-10-13 10:07:35] iteration 704/ 2384 | consumed samples: 22528 | elapsed time per iteration (ms): 224074.9 | throughput per GPU (TFLOP/s/GPU): 25.3 | MFU 8.12% | FLOPs estimator: workload-aware | logical tokens/s: 74873.0 | valid tokens/s: 74873.0 | physical tokens/s: 75313.1 | loss tokens/s: 61635.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.192227E-01 | lm-global-supervised-tokens: 1.381101E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.236128E-02 | lm-token-buffer-mib: 2.327318E+02 | global_load_balancing_loss: 1.001210E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11199.219 | params rms: 0.060640 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:34:05.789394 | finish at 2026-10-17 18:41:41 + [2026-10-13 10:11:18] iteration 705/ 2384 | consumed samples: 22560 | elapsed time per iteration (ms): 222577.6 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.84% | FLOPs estimator: workload-aware | logical tokens/s: 75376.9 | valid tokens/s: 75376.9 | physical tokens/s: 75982.8 | loss tokens/s: 59490.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.095379E-01 | lm-global-supervised-tokens: 1.324124E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.618122E-02 | lm-token-buffer-mib: 2.340089E+02 | global_load_balancing_loss: 1.000076E+00 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11199.171 | params rms: 0.060640 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:48:27.719950 | finish at 2026-10-17 17:59:46 + [2026-10-13 10:15:03] iteration 706/ 2384 | consumed samples: 22592 | elapsed time per iteration (ms): 225208.2 | throughput per GPU (TFLOP/s/GPU): 24.4 | MFU 7.81% | FLOPs estimator: workload-aware | logical tokens/s: 74496.4 | valid tokens/s: 74496.4 | physical tokens/s: 75021.9 | loss tokens/s: 53590.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.328448E-01 | lm-global-supervised-tokens: 1.206892E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.420769E-02 | lm-token-buffer-mib: 2.343698E+02 | global_load_balancing_loss: 9.988647E-01 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11199.123 | params rms: 0.060640 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:58:19.287151 | finish at 2026-10-17 19:13:22 + [2026-10-13 10:18:47] iteration 707/ 2384 | consumed samples: 22624 | elapsed time per iteration (ms): 224010.4 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.75% | FLOPs estimator: workload-aware | logical tokens/s: 74894.5 | valid tokens/s: 74894.5 | physical tokens/s: 75343.9 | loss tokens/s: 58844.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.456110E-01 | lm-global-supervised-tokens: 1.318182E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.333982E-02 | lm-token-buffer-mib: 2.332017E+02 | global_load_balancing_loss: 9.986701E-01 | loss scale: 1.0 | grad norm: 0.129 | params norm: 11199.075 | params rms: 0.060639 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:21:05.490074 | finish at 2026-10-17 18:39:53 + [2026-10-13 10:22:32] iteration 708/ 2384 | consumed samples: 22656 | elapsed time per iteration (ms): 224841.9 | throughput per GPU (TFLOP/s/GPU): 24.1 | MFU 7.71% | FLOPs estimator: workload-aware | logical tokens/s: 74617.7 | valid tokens/s: 74617.7 | physical tokens/s: 75358.5 | loss tokens/s: 57220.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 9.188565E-01 | lm-global-supervised-tokens: 1.286549E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.512305E-02 | lm-token-buffer-mib: 2.359583E+02 | global_load_balancing_loss: 9.987684E-01 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11199.027 | params rms: 0.060639 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:40:34.989421 | finish at 2026-10-17 19:03:07 + [2026-10-13 10:26:14] iteration 709/ 2384 | consumed samples: 22688 | elapsed time per iteration (ms): 222053.9 | throughput per GPU (TFLOP/s/GPU): 19.7 | MFU 6.30% | FLOPs estimator: workload-aware | logical tokens/s: 75554.7 | valid tokens/s: 75554.7 | physical tokens/s: 76592.3 | loss tokens/s: 58275.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.716498E-01 | lm-global-supervised-tokens: 1.294030E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.318113E-02 | lm-token-buffer-mib: 2.363135E+02 | global_load_balancing_loss: 1.000369E+00 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11198.978 | params rms: 0.060639 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:19:00.202236 | finish at 2026-10-17 17:45:14 + [2026-10-13 10:30:19] iteration 710/ 2384 | consumed samples: 22720 | elapsed time per iteration (ms): 244722.1 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.67% | FLOPs estimator: workload-aware | logical tokens/s: 68556.1 | valid tokens/s: 68556.1 | physical tokens/s: 69457.4 | loss tokens/s: 53042.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.472138E-01 | lm-global-supervised-tokens: 1.298060E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.769137E-02 | lm-token-buffer-mib: 2.353078E+02 | global_load_balancing_loss: 1.001025E+00 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11198.930 | params rms: 0.060639 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:47:44.782661 | finish at 2026-10-18 04:18:03 + [2026-10-13 10:34:02] iteration 711/ 2384 | consumed samples: 22752 | elapsed time per iteration (ms): 223310.3 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.97% | FLOPs estimator: workload-aware | logical tokens/s: 75129.6 | valid tokens/s: 75129.6 | physical tokens/s: 75653.2 | loss tokens/s: 54978.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.389422E-01 | lm-global-supervised-tokens: 1.227731E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.660290E-02 | lm-token-buffer-mib: 2.338969E+02 | global_load_balancing_loss: 1.002148E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11198.881 | params rms: 0.060638 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:46:38.106160 | finish at 2026-10-17 18:20:40 + [2026-10-13 10:37:46] iteration 712/ 2384 | consumed samples: 22784 | elapsed time per iteration (ms): 223670.3 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.83% | FLOPs estimator: workload-aware | logical tokens/s: 75008.6 | valid tokens/s: 75008.6 | physical tokens/s: 75876.0 | loss tokens/s: 59111.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.731321E-01 | lm-global-supervised-tokens: 1.322144E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.480468E-02 | lm-token-buffer-mib: 2.346193E+02 | global_load_balancing_loss: 1.001066E+00 | loss scale: 1.0 | grad norm: 0.169 | params norm: 11198.833 | params rms: 0.060638 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:52:56.754368 | finish at 2026-10-17 18:30:42 + [2026-10-13 10:41:27] iteration 713/ 2384 | consumed samples: 22816 | elapsed time per iteration (ms): 221069.9 | throughput per GPU (TFLOP/s/GPU): 19.6 | MFU 6.27% | FLOPs estimator: workload-aware | logical tokens/s: 75890.9 | valid tokens/s: 75890.9 | physical tokens/s: 76501.5 | loss tokens/s: 60874.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.971945E-01 | lm-global-supervised-tokens: 1.345760E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.700819E-02 | lm-token-buffer-mib: 2.333793E+02 | global_load_balancing_loss: 9.997233E-01 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11198.783 | params rms: 0.060638 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:36:47.874671 | finish at 2026-10-17 17:18:15 + [2026-10-13 10:45:08] iteration 714/ 2384 | consumed samples: 22848 | elapsed time per iteration (ms): 220862.6 | throughput per GPU (TFLOP/s/GPU): 18.7 | MFU 6.01% | FLOPs estimator: workload-aware | logical tokens/s: 75962.1 | valid tokens/s: 75962.1 | physical tokens/s: 76902.3 | loss tokens/s: 60354.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.294100E-01 | lm-global-supervised-tokens: 1.332994E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.416495E-02 | lm-token-buffer-mib: 2.349800E+02 | global_load_balancing_loss: 1.000277E+00 | loss scale: 1.0 | grad norm: 0.208 | params norm: 11198.731 | params rms: 0.060637 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:27:20.511091 | finish at 2026-10-17 17:12:28 + [2026-10-13 10:48:50] iteration 715/ 2384 | consumed samples: 22880 | elapsed time per iteration (ms): 222852.8 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.08% | FLOPs estimator: workload-aware | logical tokens/s: 75283.8 | valid tokens/s: 75283.8 | physical tokens/s: 76208.3 | loss tokens/s: 60103.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.101022E-01 | lm-global-supervised-tokens: 1.339412E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.431685E-02 | lm-token-buffer-mib: 2.340569E+02 | global_load_balancing_loss: 9.998237E-01 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11198.680 | params rms: 0.060637 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:19:01.344110 | finish at 2026-10-17 18:07:52 + [2026-10-13 10:52:34] iteration 716/ 2384 | consumed samples: 22912 | elapsed time per iteration (ms): 223594.7 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.41% | FLOPs estimator: workload-aware | logical tokens/s: 75034.0 | valid tokens/s: 75034.0 | physical tokens/s: 76134.6 | loss tokens/s: 54991.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.232859E-01 | lm-global-supervised-tokens: 1.229583E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 6.634140E-02 | lm-token-buffer-mib: 2.371725E+02 | global_load_balancing_loss: 1.000507E+00 | loss scale: 1.0 | grad norm: 0.141 | params norm: 11198.627 | params rms: 0.060637 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:35:55.944652 | finish at 2026-10-17 18:28:30 + [2026-10-13 10:56:17] iteration 717/ 2384 | consumed samples: 22944 | elapsed time per iteration (ms): 223120.7 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.31% | FLOPs estimator: workload-aware | logical tokens/s: 75193.2 | valid tokens/s: 75193.2 | physical tokens/s: 76054.6 | loss tokens/s: 61481.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.793550E-01 | lm-global-supervised-tokens: 1.371771E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.292925E-02 | lm-token-buffer-mib: 2.348579E+02 | global_load_balancing_loss: 1.000850E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11198.577 | params rms: 0.060637 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:19:02.153049 | finish at 2026-10-17 18:15:19 + [2026-10-13 11:00:01] iteration 718/ 2384 | consumed samples: 22976 | elapsed time per iteration (ms): 224120.1 | throughput per GPU (TFLOP/s/GPU): 24.3 | MFU 7.77% | FLOPs estimator: workload-aware | logical tokens/s: 74858.0 | valid tokens/s: 74858.0 | physical tokens/s: 75286.2 | loss tokens/s: 51802.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.849596E-01 | lm-global-supervised-tokens: 1.160992E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.526359E-02 | lm-token-buffer-mib: 2.336387E+02 | global_load_balancing_loss: 1.001067E+00 | loss scale: 1.0 | grad norm: 0.211 | params norm: 11198.527 | params rms: 0.060636 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:43:04.083061 | finish at 2026-10-17 18:43:05 + [2026-10-13 11:03:46] iteration 719/ 2384 | consumed samples: 23008 | elapsed time per iteration (ms): 224795.5 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.80% | FLOPs estimator: workload-aware | logical tokens/s: 74633.2 | valid tokens/s: 74633.2 | physical tokens/s: 75603.2 | loss tokens/s: 58189.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.666698E-01 | lm-global-supervised-tokens: 1.308076E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.690933E-02 | lm-token-buffer-mib: 2.352645E+02 | global_load_balancing_loss: 1.000143E+00 | loss scale: 1.0 | grad norm: 0.096 | params norm: 11198.478 | params rms: 0.060636 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:58:04.469855 | finish at 2026-10-17 19:01:51 + [2026-10-13 11:07:55] iteration 720/ 2384 | consumed samples: 23040 | elapsed time per iteration (ms): 248995.4 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.63% | FLOPs estimator: workload-aware | logical tokens/s: 67379.6 | valid tokens/s: 67379.6 | physical tokens/s: 68130.2 | loss tokens/s: 52031.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.237492E-01 | lm-global-supervised-tokens: 1.295557E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.356264E-02 | lm-token-buffer-mib: 2.345708E+02 | global_load_balancing_loss: 1.003127E+00 | loss scale: 1.0 | grad norm: 0.136 | params norm: 11198.430 | params rms: 0.060636 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:05:28.349487 | finish at 2026-10-18 06:13:23 +times across ranks (ms): + save-checkpoint: + rank 0: 24106.88 + rank 1: 24106.91 + rank 2: 24106.89 + rank 3: 24106.92 + rank 4: 24106.89 + rank 5: 24106.91 + rank 6: 24106.91 + rank 7: 24106.88 + rank 8: 24106.98 + rank 9: 24107.03 + rank 10: 24106.97 + rank 11: 24106.95 + rank 12: 24106.94 + rank 13: 24106.94 + rank 14: 24106.96 + rank 15: 24106.94 + rank 16: 24107.07 + rank 17: 24107.03 + rank 18: 24107.05 + rank 19: 24107.02 + rank 20: 24107.04 + rank 21: 24107.06 + rank 22: 24107.03 + rank 23: 24107.04 + rank 24: 24107.53 + rank 25: 24107.32 + rank 26: 24107.27 + rank 27: 24107.33 + rank 28: 24107.36 + rank 29: 24107.33 + rank 30: 24107.36 + rank 31: 24107.33 + rank 32: 24106.70 + rank 33: 24106.74 + rank 34: 24106.68 + rank 35: 24106.73 + rank 36: 24106.67 + rank 37: 24106.68 + rank 38: 24106.66 + rank 39: 24106.67 + rank 40: 24107.16 + rank 41: 24107.15 + rank 42: 24107.15 + rank 43: 24107.17 + rank 44: 24107.16 + rank 45: 24107.15 + rank 46: 24107.15 + rank 47: 24107.15 + rank 48: 24106.26 + rank 49: 24106.27 + rank 50: 24106.27 + rank 51: 24106.24 + rank 52: 24106.25 + rank 53: 24106.27 + rank 54: 24106.24 + rank 55: 24106.24 + rank 56: 24107.18 + rank 57: 24107.20 + rank 58: 24107.19 + rank 59: 24107.16 + rank 60: 24107.18 + rank 61: 24107.20 + rank 62: 24107.16 + rank 63: 24107.19 + [2026-10-13 11:11:59] iteration 721/ 2384 | consumed samples: 23072 | elapsed time per iteration (ms): 219434.9 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.07% | FLOPs estimator: workload-aware | logical tokens/s: 76456.4 | valid tokens/s: 76456.3 | physical tokens/s: 77223.5 | loss tokens/s: 60062.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.131179E-01 | lm-global-supervised-tokens: 1.317983E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.700390E-01 | lm-token-buffer-mib: 2.336890E+02 | global_load_balancing_loss: 9.983805E-01 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11198.385 | params rms: 0.060636 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:22:00.272081 | finish at 2026-10-17 16:33:59 + [2026-10-13 11:15:44] iteration 722/ 2384 | consumed samples: 23104 | elapsed time per iteration (ms): 225004.2 | throughput per GPU (TFLOP/s/GPU): 25.5 | MFU 8.16% | FLOPs estimator: workload-aware | logical tokens/s: 74563.8 | valid tokens/s: 74563.8 | physical tokens/s: 74882.0 | loss tokens/s: 54743.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.054846E-01 | lm-global-supervised-tokens: 1.231754E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.523003E-02 | lm-token-buffer-mib: 2.326988E+02 | global_load_balancing_loss: 1.000166E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11198.335 | params rms: 0.060635 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:52:36.906271 | finish at 2026-10-17 19:08:21 + [2026-10-13 11:19:28] iteration 723/ 2384 | consumed samples: 23136 | elapsed time per iteration (ms): 224145.6 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.41% | FLOPs estimator: workload-aware | logical tokens/s: 74849.5 | valid tokens/s: 74849.5 | physical tokens/s: 75745.6 | loss tokens/s: 61029.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.113627E-01 | lm-global-supervised-tokens: 1.367944E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.281968E-02 | lm-token-buffer-mib: 2.353433E+02 | global_load_balancing_loss: 9.989891E-01 | loss scale: 1.0 | grad norm: 0.149 | params norm: 11198.283 | params rms: 0.060635 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:25:05.867871 | finish at 2026-10-17 18:44:34 + [2026-10-13 11:23:10] iteration 724/ 2384 | consumed samples: 23168 | elapsed time per iteration (ms): 222107.7 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.37% | FLOPs estimator: workload-aware | logical tokens/s: 75536.2 | valid tokens/s: 75536.2 | physical tokens/s: 75838.4 | loss tokens/s: 55195.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.442790E-01 | lm-global-supervised-tokens: 1.225943E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.558326E-02 | lm-token-buffer-mib: 2.331771E+02 | global_load_balancing_loss: 9.996031E-01 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11198.233 | params rms: 0.060635 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:24:58.818197 | finish at 2026-10-17 17:48:09 + [2026-10-13 11:26:54] iteration 725/ 2384 | consumed samples: 23200 | elapsed time per iteration (ms): 223742.4 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.80% | FLOPs estimator: workload-aware | logical tokens/s: 74984.5 | valid tokens/s: 74984.5 | physical tokens/s: 75899.3 | loss tokens/s: 57287.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.869190E-01 | lm-global-supervised-tokens: 1.281773E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.198543E-02 | lm-token-buffer-mib: 2.363057E+02 | global_load_balancing_loss: 9.994869E-01 | loss scale: 1.0 | grad norm: 0.164 | params norm: 11198.185 | params rms: 0.060635 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:06:28.568311 | finish at 2026-10-17 18:33:22 + [2026-10-13 11:30:37] iteration 726/ 2384 | consumed samples: 23232 | elapsed time per iteration (ms): 223016.4 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.42% | FLOPs estimator: workload-aware | logical tokens/s: 75228.4 | valid tokens/s: 75228.4 | physical tokens/s: 76054.0 | loss tokens/s: 57891.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.906678E-01 | lm-global-supervised-tokens: 1.291074E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.185828E-02 | lm-token-buffer-mib: 2.362969E+02 | global_load_balancing_loss: 1.001128E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11198.136 | params rms: 0.060634 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:42:41.167646 | finish at 2026-10-17 18:13:18 + [2026-10-13 11:34:23] iteration 727/ 2384 | consumed samples: 23264 | elapsed time per iteration (ms): 226271.4 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.79% | FLOPs estimator: workload-aware | logical tokens/s: 74146.2 | valid tokens/s: 74146.2 | physical tokens/s: 75514.9 | loss tokens/s: 57495.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.687253E-01 | lm-global-supervised-tokens: 1.300946E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.051748E-02 | lm-token-buffer-mib: 2.352827E+02 | global_load_balancing_loss: 1.001685E+00 | loss scale: 1.0 | grad norm: 0.191 | params norm: 11198.085 | params rms: 0.060634 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:08:51.640623 | finish at 2026-10-17 19:43:15 + [2026-10-13 11:38:07] iteration 728/ 2384 | consumed samples: 23296 | elapsed time per iteration (ms): 224437.4 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.55% | FLOPs estimator: workload-aware | logical tokens/s: 74752.3 | valid tokens/s: 74752.3 | physical tokens/s: 75119.9 | loss tokens/s: 54467.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.286328E-01 | lm-global-supervised-tokens: 1.222454E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.187223E-02 | lm-token-buffer-mib: 2.336187E+02 | global_load_balancing_loss: 1.002169E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11198.033 | params rms: 0.060634 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:14:28.256395 | finish at 2026-10-17 18:52:36 + [2026-10-13 11:41:55] iteration 729/ 2384 | consumed samples: 23328 | elapsed time per iteration (ms): 227524.4 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.15% | FLOPs estimator: workload-aware | logical tokens/s: 73738.1 | valid tokens/s: 73738.1 | physical tokens/s: 75084.3 | loss tokens/s: 55639.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.038746E-01 | lm-global-supervised-tokens: 1.265925E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.294890E-02 | lm-token-buffer-mib: 2.377928E+02 | global_load_balancing_loss: 1.001788E+00 | loss scale: 1.0 | grad norm: 0.171 | params norm: 11197.982 | params rms: 0.060633 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:35:52.937629 | finish at 2026-10-17 20:17:48 + [2026-10-13 11:46:01] iteration 730/ 2384 | consumed samples: 23360 | elapsed time per iteration (ms): 246313.5 | throughput per GPU (TFLOP/s/GPU): 19.4 | MFU 6.21% | FLOPs estimator: workload-aware | logical tokens/s: 68113.1 | valid tokens/s: 68113.1 | physical tokens/s: 68655.8 | loss tokens/s: 51722.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.188626E-01 | lm-global-supervised-tokens: 1.273986E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.065022E-02 | lm-token-buffer-mib: 2.349288E+02 | global_load_balancing_loss: 1.000727E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11197.931 | params rms: 0.060633 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:10:02.540710 | finish at 2026-10-18 04:56:04 + [2026-10-13 11:49:42] iteration 731/ 2384 | consumed samples: 23392 | elapsed time per iteration (ms): 221011.4 | throughput per GPU (TFLOP/s/GPU): 19.1 | MFU 6.13% | FLOPs estimator: workload-aware | logical tokens/s: 75910.9 | valid tokens/s: 75910.9 | physical tokens/s: 76902.6 | loss tokens/s: 60784.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.286929E-01 | lm-global-supervised-tokens: 1.343396E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.473890E-02 | lm-token-buffer-mib: 2.341722E+02 | global_load_balancing_loss: 9.987827E-01 | loss scale: 1.0 | grad norm: 0.218 | params norm: 11197.881 | params rms: 0.060633 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:28:51.830775 | finish at 2026-10-17 17:18:34 + [2026-10-13 11:53:28] iteration 732/ 2384 | consumed samples: 23424 | elapsed time per iteration (ms): 225463.1 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.01% | FLOPs estimator: workload-aware | logical tokens/s: 74412.1 | valid tokens/s: 74412.1 | physical tokens/s: 75041.8 | loss tokens/s: 57702.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.412050E-01 | lm-global-supervised-tokens: 1.300986E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.164926E-02 | lm-token-buffer-mib: 2.353239E+02 | global_load_balancing_loss: 1.000020E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11197.833 | params rms: 0.060633 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:27:45.047824 | finish at 2026-10-17 19:21:13 + [2026-10-13 11:57:09] iteration 733/ 2384 | consumed samples: 23456 | elapsed time per iteration (ms): 221336.8 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.35% | FLOPs estimator: workload-aware | logical tokens/s: 75799.3 | valid tokens/s: 75799.3 | physical tokens/s: 76384.7 | loss tokens/s: 55648.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.174643E-01 | lm-global-supervised-tokens: 1.231695E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.134514E-02 | lm-token-buffer-mib: 2.333158E+02 | global_load_balancing_loss: 9.994841E-01 | loss scale: 1.0 | grad norm: 0.291 | params norm: 11197.789 | params rms: 0.060632 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:30:27.100262 | finish at 2026-10-17 17:27:36 + [2026-10-13 12:00:57] iteration 734/ 2384 | consumed samples: 23488 | elapsed time per iteration (ms): 227701.8 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.40% | FLOPs estimator: workload-aware | logical tokens/s: 73680.5 | valid tokens/s: 73680.5 | physical tokens/s: 74914.0 | loss tokens/s: 53343.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.363513E-01 | lm-global-supervised-tokens: 1.214649E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.471889E-02 | lm-token-buffer-mib: 2.362708E+02 | global_load_balancing_loss: 1.000077E+00 | loss scale: 1.0 | grad norm: 0.165 | params norm: 11197.740 | params rms: 0.060632 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:21:47.908809 | finish at 2026-10-17 20:22:45 + [2026-10-13 12:04:40] iteration 735/ 2384 | consumed samples: 23520 | elapsed time per iteration (ms): 223521.8 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.23% | FLOPs estimator: workload-aware | logical tokens/s: 75058.4 | valid tokens/s: 75058.4 | physical tokens/s: 75640.9 | loss tokens/s: 52603.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.732439E-01 | lm-global-supervised-tokens: 1.175808E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.499839E-02 | lm-token-buffer-mib: 2.338167E+02 | global_load_balancing_loss: 1.000041E+00 | loss scale: 1.0 | grad norm: 0.147 | params norm: 11197.691 | params rms: 0.060632 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:23:07.476575 | finish at 2026-10-17 18:27:48 + [2026-10-13 12:08:24] iteration 736/ 2384 | consumed samples: 23552 | elapsed time per iteration (ms): 223567.2 | throughput per GPU (TFLOP/s/GPU): 26.0 | MFU 8.34% | FLOPs estimator: workload-aware | logical tokens/s: 75043.1 | valid tokens/s: 75043.1 | physical tokens/s: 75603.5 | loss tokens/s: 61166.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.765692E-01 | lm-global-supervised-tokens: 1.367485E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.100983E-02 | lm-token-buffer-mib: 2.337174E+02 | global_load_balancing_loss: 1.001401E+00 | loss scale: 1.0 | grad norm: 0.122 | params norm: 11197.645 | params rms: 0.060632 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:20:38.674000 | finish at 2026-10-17 18:29:03 + [2026-10-13 12:12:09] iteration 737/ 2384 | consumed samples: 23584 | elapsed time per iteration (ms): 224747.3 | throughput per GPU (TFLOP/s/GPU): 26.3 | MFU 8.44% | FLOPs estimator: workload-aware | logical tokens/s: 74649.1 | valid tokens/s: 74649.1 | physical tokens/s: 75158.4 | loss tokens/s: 50268.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.437567E-01 | lm-global-supervised-tokens: 1.129766E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.595661E-02 | lm-token-buffer-mib: 2.347937E+02 | global_load_balancing_loss: 1.001428E+00 | loss scale: 1.0 | grad norm: 0.184 | params norm: 11197.599 | params rms: 0.060631 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:49:18.875203 | finish at 2026-10-17 19:01:28 + [2026-10-13 12:15:53] iteration 738/ 2384 | consumed samples: 23616 | elapsed time per iteration (ms): 224707.1 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.82% | FLOPs estimator: workload-aware | logical tokens/s: 74662.4 | valid tokens/s: 74662.3 | physical tokens/s: 75569.2 | loss tokens/s: 57563.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.757092E-01 | lm-global-supervised-tokens: 1.293494E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.398426E-02 | lm-token-buffer-mib: 2.346701E+02 | global_load_balancing_loss: 1.001111E+00 | loss scale: 1.0 | grad norm: 0.241 | params norm: 11197.549 | params rms: 0.060631 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:44:27.950034 | finish at 2026-10-17 19:00:21 + [2026-10-13 12:19:39] iteration 739/ 2384 | consumed samples: 23648 | elapsed time per iteration (ms): 225293.5 | throughput per GPU (TFLOP/s/GPU): 24.5 | MFU 7.85% | FLOPs estimator: workload-aware | logical tokens/s: 74468.1 | valid tokens/s: 74468.1 | physical tokens/s: 75579.0 | loss tokens/s: 54925.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.381570E-01 | lm-global-supervised-tokens: 1.237426E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.980169E-02 | lm-token-buffer-mib: 2.377553E+02 | global_load_balancing_loss: 9.998122E-01 | loss scale: 1.0 | grad norm: 0.132 | params norm: 11197.503 | params rms: 0.060631 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:56:47.864673 | finish at 2026-10-17 19:16:26 + [2026-10-13 12:23:47] iteration 740/ 2384 | consumed samples: 23680 | elapsed time per iteration (ms): 248576.6 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.13% | FLOPs estimator: workload-aware | logical tokens/s: 67493.0 | valid tokens/s: 67493.0 | physical tokens/s: 68666.7 | loss tokens/s: 50134.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.509976E-01 | lm-global-supervised-tokens: 1.246236E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.265719E-02 | lm-token-buffer-mib: 2.371351E+02 | global_load_balancing_loss: 1.000885E+00 | loss scale: 1.0 | grad norm: 0.319 | params norm: 11197.457 | params rms: 0.060631 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:30:59.885056 | finish at 2026-10-18 05:54:47 + [2026-10-13 12:27:34] iteration 741/ 2384 | consumed samples: 23712 | elapsed time per iteration (ms): 226415.6 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.74% | FLOPs estimator: workload-aware | logical tokens/s: 74099.0 | valid tokens/s: 74099.0 | physical tokens/s: 74837.2 | loss tokens/s: 54438.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.338905E-01 | lm-global-supervised-tokens: 1.232581E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.586886E-02 | lm-token-buffer-mib: 2.356082E+02 | global_load_balancing_loss: 1.000672E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11197.413 | params rms: 0.060630 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:20:00.883392 | finish at 2026-10-17 19:47:35 + [2026-10-13 12:31:18] iteration 742/ 2384 | consumed samples: 23744 | elapsed time per iteration (ms): 224499.5 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.41% | FLOPs estimator: workload-aware | logical tokens/s: 74731.5 | valid tokens/s: 74731.5 | physical tokens/s: 75564.4 | loss tokens/s: 55099.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.369270E-01 | lm-global-supervised-tokens: 1.236970E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.502270E-02 | lm-token-buffer-mib: 2.341598E+02 | global_load_balancing_loss: 1.000176E+00 | loss scale: 1.0 | grad norm: 0.205 | params norm: 11197.369 | params rms: 0.060630 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:23:48.200200 | finish at 2026-10-17 18:55:06 + [2026-10-13 12:35:06] iteration 743/ 2384 | consumed samples: 23776 | elapsed time per iteration (ms): 227523.6 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.63% | FLOPs estimator: workload-aware | logical tokens/s: 73738.4 | valid tokens/s: 73738.4 | physical tokens/s: 74725.4 | loss tokens/s: 52631.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.442865E-01 | lm-global-supervised-tokens: 1.197499E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.264757E-02 | lm-token-buffer-mib: 2.369137E+02 | global_load_balancing_loss: 1.001190E+00 | loss scale: 1.0 | grad norm: 0.121 | params norm: 11197.324 | params rms: 0.060630 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:42:46.203118 | finish at 2026-10-17 20:17:52 + [2026-10-13 12:38:46] iteration 744/ 2384 | consumed samples: 23808 | elapsed time per iteration (ms): 219969.7 | throughput per GPU (TFLOP/s/GPU): 18.0 | MFU 5.77% | FLOPs estimator: workload-aware | logical tokens/s: 76270.6 | valid tokens/s: 76270.6 | physical tokens/s: 77233.3 | loss tokens/s: 60900.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.986225E-01 | lm-global-supervised-tokens: 1.339621E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.522762E-02 | lm-token-buffer-mib: 2.337983E+02 | global_load_balancing_loss: 1.000071E+00 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11197.282 | params rms: 0.060630 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:12:30.235434 | finish at 2026-10-17 16:51:16 + [2026-10-13 12:42:34] iteration 745/ 2384 | consumed samples: 23840 | elapsed time per iteration (ms): 227921.1 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.64% | FLOPs estimator: workload-aware | logical tokens/s: 73609.7 | valid tokens/s: 73609.7 | physical tokens/s: 74668.9 | loss tokens/s: 55525.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.064725E-01 | lm-global-supervised-tokens: 1.265552E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.961523E-02 | lm-token-buffer-mib: 2.366193E+02 | global_load_balancing_loss: 9.994727E-01 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11197.242 | params rms: 0.060629 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:46:02.736273 | finish at 2026-10-17 20:28:36 + [2026-10-13 12:46:14] iteration 746/ 2384 | consumed samples: 23872 | elapsed time per iteration (ms): 220009.2 | throughput per GPU (TFLOP/s/GPU): 23.9 | MFU 7.67% | FLOPs estimator: workload-aware | logical tokens/s: 76256.7 | valid tokens/s: 76256.7 | physical tokens/s: 76816.6 | loss tokens/s: 58825.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.767231E-01 | lm-global-supervised-tokens: 1.294215E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.324082E-02 | lm-token-buffer-mib: 2.328476E+02 | global_load_balancing_loss: 1.000100E+00 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11197.202 | params rms: 0.060629 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:06:15.015083 | finish at 2026-10-17 16:52:29 + [2026-10-13 12:49:57] iteration 747/ 2384 | consumed samples: 23904 | elapsed time per iteration (ms): 223032.3 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.73% | FLOPs estimator: workload-aware | logical tokens/s: 75223.2 | valid tokens/s: 75223.2 | physical tokens/s: 75839.9 | loss tokens/s: 58311.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.540918E-01 | lm-global-supervised-tokens: 1.300536E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.563115E-02 | lm-token-buffer-mib: 2.329986E+02 | global_load_balancing_loss: 1.000106E+00 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11197.164 | params rms: 0.060629 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:25:03.796644 | finish at 2026-10-17 18:15:00 + [2026-10-13 12:53:42] iteration 748/ 2384 | consumed samples: 23936 | elapsed time per iteration (ms): 225340.4 | throughput per GPU (TFLOP/s/GPU): 24.8 | MFU 7.95% | FLOPs estimator: workload-aware | logical tokens/s: 74452.5 | valid tokens/s: 74452.5 | physical tokens/s: 74765.5 | loss tokens/s: 55870.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.263192E-01 | lm-global-supervised-tokens: 1.258979E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.211366E-02 | lm-token-buffer-mib: 2.332303E+02 | global_load_balancing_loss: 1.000446E+00 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11197.124 | params rms: 0.060629 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:24:16.893978 | finish at 2026-10-17 19:17:59 + [2026-10-13 12:57:22] iteration 749/ 2384 | consumed samples: 23968 | elapsed time per iteration (ms): 220244.0 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.23% | FLOPs estimator: workload-aware | logical tokens/s: 76175.6 | valid tokens/s: 76175.6 | physical tokens/s: 76630.6 | loss tokens/s: 56342.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.777633E-01 | lm-global-supervised-tokens: 1.240917E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.222098E-02 | lm-token-buffer-mib: 2.331569E+02 | global_load_balancing_loss: 1.000739E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11197.079 | params rms: 0.060629 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:01:38.962930 | finish at 2026-10-17 16:59:01 + [2026-10-13 13:01:25] iteration 750/ 2384 | consumed samples: 24000 | elapsed time per iteration (ms): 243252.7 | throughput per GPU (TFLOP/s/GPU): 19.3 | MFU 6.18% | FLOPs estimator: workload-aware | logical tokens/s: 68970.2 | valid tokens/s: 68970.2 | physical tokens/s: 69417.6 | loss tokens/s: 54068.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.592895E-01 | lm-global-supervised-tokens: 1.315225E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.270236E-02 | lm-token-buffer-mib: 2.338841E+02 | global_load_balancing_loss: 1.000544E+00 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11197.036 | params rms: 0.060628 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:24:34.984409 | finish at 2026-10-18 03:26:00 + [2026-10-13 13:05:09] iteration 751/ 2384 | consumed samples: 24032 | elapsed time per iteration (ms): 223104.9 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.73% | FLOPs estimator: workload-aware | logical tokens/s: 75198.7 | valid tokens/s: 75198.7 | physical tokens/s: 76996.5 | loss tokens/s: 58383.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.668917E-01 | lm-global-supervised-tokens: 1.302558E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.865866E-02 | lm-token-buffer-mib: 2.350897E+02 | global_load_balancing_loss: 1.000197E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11196.998 | params rms: 0.060628 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:12:10.380156 | finish at 2026-10-17 18:17:19 + [2026-10-13 13:09:34] iteration 752/ 2384 | consumed samples: 24064 | elapsed time per iteration (ms): 265194.9 | throughput per GPU (TFLOP/s/GPU): 17.9 | MFU 5.75% | FLOPs estimator: workload-aware | logical tokens/s: 63263.7 | valid tokens/s: 63263.7 | physical tokens/s: 64878.7 | loss tokens/s: 48533.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.181992E-01 | lm-global-supervised-tokens: 1.287079E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.247195E-02 | lm-token-buffer-mib: 2.373016E+02 | global_load_balancing_loss: 1.000388E+00 | loss scale: 1.0 | grad norm: 0.107 | params norm: 11196.958 | params rms: 0.060628 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:13:18.109543 | finish at 2026-10-18 13:22:52 + [2026-10-13 13:13:19] iteration 753/ 2384 | consumed samples: 24096 | elapsed time per iteration (ms): 225031.3 | throughput per GPU (TFLOP/s/GPU): 19.6 | MFU 6.27% | FLOPs estimator: workload-aware | logical tokens/s: 74554.9 | valid tokens/s: 74554.9 | physical tokens/s: 75859.0 | loss tokens/s: 59078.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.619164E-01 | lm-global-supervised-tokens: 1.329455E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.505944E-02 | lm-token-buffer-mib: 2.377166E+02 | global_load_balancing_loss: 9.996926E-01 | loss scale: 1.0 | grad norm: 0.182 | params norm: 11196.916 | params rms: 0.060628 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:57:06.088519 | finish at 2026-10-17 19:10:25 + [2026-10-13 13:17:10] iteration 754/ 2384 | consumed samples: 24128 | elapsed time per iteration (ms): 230986.7 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.01% | FLOPs estimator: workload-aware | logical tokens/s: 72632.7 | valid tokens/s: 72632.7 | physical tokens/s: 73007.4 | loss tokens/s: 52200.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.655586E-01 | lm-global-supervised-tokens: 1.205771E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.507682E-02 | lm-token-buffer-mib: 2.331926E+02 | global_load_balancing_loss: 1.001186E+00 | loss scale: 1.0 | grad norm: 0.185 | params norm: 11196.873 | params rms: 0.060627 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:35:08.259692 | finish at 2026-10-17 21:52:18 + [2026-10-13 13:20:52] iteration 755/ 2384 | consumed samples: 24160 | elapsed time per iteration (ms): 222421.7 | throughput per GPU (TFLOP/s/GPU): 23.8 | MFU 7.63% | FLOPs estimator: workload-aware | logical tokens/s: 75429.7 | valid tokens/s: 75429.7 | physical tokens/s: 75967.6 | loss tokens/s: 60390.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.259463E-01 | lm-global-supervised-tokens: 1.343223E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.454423E-02 | lm-token-buffer-mib: 2.323969E+02 | global_load_balancing_loss: 1.000505E+00 | loss scale: 1.0 | grad norm: 0.267 | params norm: 11196.829 | params rms: 0.060627 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:38:44.905802 | finish at 2026-10-17 17:59:37 + [2026-10-13 13:24:35] iteration 756/ 2384 | consumed samples: 24192 | elapsed time per iteration (ms): 222991.8 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.62% | FLOPs estimator: workload-aware | logical tokens/s: 75236.8 | valid tokens/s: 75236.8 | physical tokens/s: 75716.2 | loss tokens/s: 55066.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.411295E-01 | lm-global-supervised-tokens: 1.227941E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.658393E-02 | lm-token-buffer-mib: 2.334571E+02 | global_load_balancing_loss: 9.997580E-01 | loss scale: 1.0 | grad norm: 0.105 | params norm: 11196.782 | params rms: 0.060627 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:50:30.593068 | finish at 2026-10-17 18:15:06 + [2026-10-13 13:28:17] iteration 757/ 2384 | consumed samples: 24224 | elapsed time per iteration (ms): 221888.3 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.15% | FLOPs estimator: workload-aware | logical tokens/s: 75610.9 | valid tokens/s: 75610.9 | physical tokens/s: 76188.8 | loss tokens/s: 56970.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.393040E-01 | lm-global-supervised-tokens: 1.264118E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.524550E-02 | lm-token-buffer-mib: 2.343345E+02 | global_load_balancing_loss: 9.992833E-01 | loss scale: 1.0 | grad norm: 0.268 | params norm: 11196.739 | params rms: 0.060627 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:16:52.199613 | finish at 2026-10-17 17:45:09 + [2026-10-13 13:31:57] iteration 758/ 2384 | consumed samples: 24256 | elapsed time per iteration (ms): 219978.5 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.39% | FLOPs estimator: workload-aware | logical tokens/s: 76267.5 | valid tokens/s: 76267.5 | physical tokens/s: 77005.1 | loss tokens/s: 56526.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.109710E-01 | lm-global-supervised-tokens: 1.243453E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.039978E-02 | lm-token-buffer-mib: 2.349515E+02 | global_load_balancing_loss: 1.000049E+00 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11196.697 | params rms: 0.060626 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:21:24.975304 | finish at 2026-10-17 16:53:22 + [2026-10-13 13:35:43] iteration 759/ 2384 | consumed samples: 24288 | elapsed time per iteration (ms): 226053.8 | throughput per GPU (TFLOP/s/GPU): 20.2 | MFU 6.46% | FLOPs estimator: workload-aware | logical tokens/s: 74217.8 | valid tokens/s: 74217.7 | physical tokens/s: 75000.6 | loss tokens/s: 57454.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.673053E-01 | lm-global-supervised-tokens: 1.298776E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.149031E-02 | lm-token-buffer-mib: 2.351239E+02 | global_load_balancing_loss: 1.000857E+00 | loss scale: 1.0 | grad norm: 0.339 | params norm: 11196.656 | params rms: 0.060626 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:02:17.479413 | finish at 2026-10-17 19:38:01 + [2026-10-13 13:39:49] iteration 760/ 2384 | consumed samples: 24320 | elapsed time per iteration (ms): 245784.5 | throughput per GPU (TFLOP/s/GPU): 15.7 | MFU 5.03% | FLOPs estimator: workload-aware | logical tokens/s: 68259.8 | valid tokens/s: 68259.8 | physical tokens/s: 69140.8 | loss tokens/s: 51383.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.264051E-01 | lm-global-supervised-tokens: 1.262916E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.159384E-02 | lm-token-buffer-mib: 2.360955E+02 | global_load_balancing_loss: 1.000862E+00 | loss scale: 1.0 | grad norm: 0.112 | params norm: 11196.617 | params rms: 0.060626 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:52:34.094021 | finish at 2026-10-18 04:32:23 +times across ranks (ms): + save-checkpoint: + rank 0: 24100.20 + rank 1: 24100.22 + rank 2: 24100.23 + rank 3: 24100.24 + rank 4: 24100.21 + rank 5: 24100.22 + rank 6: 24100.29 + rank 7: 24100.21 + rank 8: 24099.53 + rank 9: 24099.54 + rank 10: 24099.54 + rank 11: 24099.55 + rank 12: 24099.52 + rank 13: 24099.52 + rank 14: 24099.52 + rank 15: 24099.52 + rank 16: 24099.67 + rank 17: 24099.66 + rank 18: 24099.66 + rank 19: 24099.69 + rank 20: 24099.66 + rank 21: 24099.67 + rank 22: 24099.65 + rank 23: 24099.68 + rank 24: 24099.79 + rank 25: 24099.80 + rank 26: 24099.79 + rank 27: 24099.81 + rank 28: 24099.80 + rank 29: 24099.87 + rank 30: 24099.82 + rank 31: 24099.79 + rank 32: 24098.63 + rank 33: 24098.65 + rank 34: 24098.69 + rank 35: 24098.69 + rank 36: 24098.65 + rank 37: 24098.69 + rank 38: 24098.65 + rank 39: 24098.67 + rank 40: 24100.98 + rank 41: 24100.97 + rank 42: 24100.94 + rank 43: 24100.97 + rank 44: 24100.95 + rank 45: 24100.97 + rank 46: 24100.96 + rank 47: 24100.93 + rank 48: 24099.60 + rank 49: 24099.59 + rank 50: 24099.61 + rank 51: 24099.60 + rank 52: 24099.59 + rank 53: 24099.61 + rank 54: 24099.61 + rank 55: 24099.59 + rank 56: 24099.34 + rank 57: 24099.34 + rank 58: 24099.36 + rank 59: 24099.33 + rank 60: 24099.34 + rank 61: 24099.42 + rank 62: 24099.35 + rank 63: 24099.38 + [2026-10-13 13:43:58] iteration 761/ 2384 | consumed samples: 24352 | elapsed time per iteration (ms): 224935.0 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.34% | FLOPs estimator: workload-aware | logical tokens/s: 74586.9 | valid tokens/s: 74586.9 | physical tokens/s: 75333.0 | loss tokens/s: 62171.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 9.367137E-01 | lm-global-supervised-tokens: 1.398459E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.308047E-01 | lm-token-buffer-mib: 2.345852E+02 | global_load_balancing_loss: 1.000834E+00 | loss scale: 1.0 | grad norm: 0.229 | params norm: 11196.582 | params rms: 0.060626 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:24:29.424421 | finish at 2026-10-17 19:08:27 + [2026-10-13 13:47:48] iteration 762/ 2384 | consumed samples: 24384 | elapsed time per iteration (ms): 229764.3 | throughput per GPU (TFLOP/s/GPU): 20.2 | MFU 6.46% | FLOPs estimator: workload-aware | logical tokens/s: 73019.1 | valid tokens/s: 73019.1 | physical tokens/s: 74766.9 | loss tokens/s: 57302.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.682671E-01 | lm-global-supervised-tokens: 1.316608E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.431967E-02 | lm-token-buffer-mib: 2.346323E+02 | global_load_balancing_loss: 1.001042E+00 | loss scale: 1.0 | grad norm: 0.355 | params norm: 11196.547 | params rms: 0.060626 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:31:17.639088 | finish at 2026-10-17 21:19:05 + [2026-10-13 13:51:28] iteration 763/ 2384 | consumed samples: 24416 | elapsed time per iteration (ms): 220530.4 | throughput per GPU (TFLOP/s/GPU): 17.7 | MFU 5.68% | FLOPs estimator: workload-aware | logical tokens/s: 76076.5 | valid tokens/s: 76076.5 | physical tokens/s: 77110.4 | loss tokens/s: 58416.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.181224E-01 | lm-global-supervised-tokens: 1.288250E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.882519E-02 | lm-token-buffer-mib: 2.348741E+02 | global_load_balancing_loss: 1.000718E+00 | loss scale: 1.0 | grad norm: 0.453 | params norm: 11196.513 | params rms: 0.060625 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:17:59.852278 | finish at 2026-10-17 17:09:28 + [2026-10-13 13:55:09] iteration 764/ 2384 | consumed samples: 24448 | elapsed time per iteration (ms): 220798.1 | throughput per GPU (TFLOP/s/GPU): 23.7 | MFU 7.59% | FLOPs estimator: workload-aware | logical tokens/s: 75984.3 | valid tokens/s: 75984.3 | physical tokens/s: 76640.4 | loss tokens/s: 58457.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.804708E-01 | lm-global-supervised-tokens: 1.290730E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.314189E-02 | lm-token-buffer-mib: 2.335327E+02 | global_load_balancing_loss: 1.001095E+00 | loss scale: 1.0 | grad norm: 0.153 | params norm: 11196.478 | params rms: 0.060625 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:21:32.938213 | finish at 2026-10-17 17:16:42 + [2026-10-13 13:58:53] iteration 765/ 2384 | consumed samples: 24480 | elapsed time per iteration (ms): 224189.5 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.15% | FLOPs estimator: workload-aware | logical tokens/s: 74834.9 | valid tokens/s: 74834.9 | physical tokens/s: 75936.1 | loss tokens/s: 55416.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.814953E-01 | lm-global-supervised-tokens: 1.242388E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.159647E-02 | lm-token-buffer-mib: 2.372473E+02 | global_load_balancing_loss: 1.000929E+00 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11196.443 | params rms: 0.060625 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:49:22.755875 | finish at 2026-10-17 18:48:16 + [2026-10-13 14:02:36] iteration 766/ 2384 | consumed samples: 24512 | elapsed time per iteration (ms): 222832.6 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.80% | FLOPs estimator: workload-aware | logical tokens/s: 75290.6 | valid tokens/s: 75290.6 | physical tokens/s: 76125.3 | loss tokens/s: 58401.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.405703E-01 | lm-global-supervised-tokens: 1.301375E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.265416E-02 | lm-token-buffer-mib: 2.335553E+02 | global_load_balancing_loss: 1.000816E+00 | loss scale: 1.0 | grad norm: 0.161 | params norm: 11196.407 | params rms: 0.060625 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:09:03.173992 | finish at 2026-10-17 18:11:39 + [2026-10-13 14:06:20] iteration 767/ 2384 | consumed samples: 24544 | elapsed time per iteration (ms): 224338.4 | throughput per GPU (TFLOP/s/GPU): 24.2 | MFU 7.76% | FLOPs estimator: workload-aware | logical tokens/s: 74785.2 | valid tokens/s: 74785.2 | physical tokens/s: 75553.6 | loss tokens/s: 58712.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.290405E-01 | lm-global-supervised-tokens: 1.317137E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.242228E-02 | lm-token-buffer-mib: 2.338105E+02 | global_load_balancing_loss: 9.992988E-01 | loss scale: 1.0 | grad norm: 0.231 | params norm: 11196.370 | params rms: 0.060625 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:45:55.162547 | finish at 2026-10-17 18:52:16 + [2026-10-13 14:10:03] iteration 768/ 2384 | consumed samples: 24576 | elapsed time per iteration (ms): 222137.0 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.14% | FLOPs estimator: workload-aware | logical tokens/s: 75526.3 | valid tokens/s: 75526.3 | physical tokens/s: 76045.3 | loss tokens/s: 57574.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.472900E-01 | lm-global-supervised-tokens: 1.278940E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.648221E-02 | lm-token-buffer-mib: 2.334325E+02 | global_load_balancing_loss: 9.999006E-01 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11196.330 | params rms: 0.060624 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:42:53.444534 | finish at 2026-10-17 17:52:56 + [2026-10-13 14:13:45] iteration 769/ 2384 | consumed samples: 24608 | elapsed time per iteration (ms): 222182.9 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.07% | FLOPs estimator: workload-aware | logical tokens/s: 75510.7 | valid tokens/s: 75510.7 | physical tokens/s: 76481.9 | loss tokens/s: 59079.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.625352E-01 | lm-global-supervised-tokens: 1.312653E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.313573E-02 | lm-token-buffer-mib: 2.354777E+02 | global_load_balancing_loss: 9.994410E-01 | loss scale: 1.0 | grad norm: 0.157 | params norm: 11196.291 | params rms: 0.060624 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:40:25.340682 | finish at 2026-10-17 17:54:10 + [2026-10-13 14:17:48] iteration 770/ 2384 | consumed samples: 24640 | elapsed time per iteration (ms): 243196.8 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.42% | FLOPs estimator: workload-aware | logical tokens/s: 68986.0 | valid tokens/s: 68986.0 | physical tokens/s: 69938.6 | loss tokens/s: 53673.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.377149E-01 | lm-global-supervised-tokens: 1.305321E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.454965E-02 | lm-token-buffer-mib: 2.348386E+02 | global_load_balancing_loss: 1.000072E+00 | loss scale: 1.0 | grad norm: 0.261 | params norm: 11196.253 | params rms: 0.060624 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:01:59.600557 | finish at 2026-10-18 03:19:47 + [2026-10-13 14:21:34] iteration 771/ 2384 | consumed samples: 24672 | elapsed time per iteration (ms): 226330.5 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.37% | FLOPs estimator: workload-aware | logical tokens/s: 74126.9 | valid tokens/s: 74126.9 | physical tokens/s: 75030.8 | loss tokens/s: 59695.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.904686E-01 | lm-global-supervised-tokens: 1.351093E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.455163E-02 | lm-token-buffer-mib: 2.351065E+02 | global_load_balancing_loss: 1.000624E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11196.217 | params rms: 0.060624 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:24:31.172848 | finish at 2026-10-17 19:46:05 + [2026-10-13 14:25:18] iteration 772/ 2384 | consumed samples: 24704 | elapsed time per iteration (ms): 223488.9 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.35% | FLOPs estimator: workload-aware | logical tokens/s: 75069.4 | valid tokens/s: 75069.4 | physical tokens/s: 75920.7 | loss tokens/s: 56454.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.369939E-01 | lm-global-supervised-tokens: 1.261700E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.120636E-02 | lm-token-buffer-mib: 2.355519E+02 | global_load_balancing_loss: 1.001468E+00 | loss scale: 1.0 | grad norm: 0.125 | params norm: 11196.183 | params rms: 0.060624 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:04:24.049064 | finish at 2026-10-17 18:29:42 + [2026-10-13 14:28:59] iteration 773/ 2384 | consumed samples: 24736 | elapsed time per iteration (ms): 221742.0 | throughput per GPU (TFLOP/s/GPU): 27.1 | MFU 8.69% | FLOPs estimator: workload-aware | logical tokens/s: 75660.8 | valid tokens/s: 75660.8 | physical tokens/s: 76377.6 | loss tokens/s: 55242.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.512100E-01 | lm-global-supervised-tokens: 1.224949E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.830237E-02 | lm-token-buffer-mib: 2.324420E+02 | global_load_balancing_loss: 1.000929E+00 | loss scale: 1.0 | grad norm: 0.167 | params norm: 11196.150 | params rms: 0.060624 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:13:46.420932 | finish at 2026-10-17 17:42:46 + [2026-10-13 14:32:42] iteration 774/ 2384 | consumed samples: 24768 | elapsed time per iteration (ms): 222458.7 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.20% | FLOPs estimator: workload-aware | logical tokens/s: 75417.1 | valid tokens/s: 75417.1 | physical tokens/s: 75988.0 | loss tokens/s: 59237.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.800561E-01 | lm-global-supervised-tokens: 1.317791E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.119737E-02 | lm-token-buffer-mib: 2.332684E+02 | global_load_balancing_loss: 1.002920E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11196.112 | params rms: 0.060623 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:29:18.493087 | finish at 2026-10-17 18:02:00 + [2026-10-13 14:36:28] iteration 775/ 2384 | consumed samples: 24800 | elapsed time per iteration (ms): 226141.7 | throughput per GPU (TFLOP/s/GPU): 19.3 | MFU 6.18% | FLOPs estimator: workload-aware | logical tokens/s: 74188.9 | valid tokens/s: 74188.9 | physical tokens/s: 74667.8 | loss tokens/s: 56367.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.644087E-01 | lm-global-supervised-tokens: 1.274698E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 5.448319E-02 | lm-token-buffer-mib: 2.343685E+02 | global_load_balancing_loss: 1.000613E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11196.075 | params rms: 0.060623 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:04:21.973481 | finish at 2026-10-17 19:40:50 + [2026-10-13 14:40:10] iteration 776/ 2384 | consumed samples: 24832 | elapsed time per iteration (ms): 221830.5 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.15% | FLOPs estimator: workload-aware | logical tokens/s: 75630.7 | valid tokens/s: 75630.7 | physical tokens/s: 76330.4 | loss tokens/s: 59325.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.838338E-01 | lm-global-supervised-tokens: 1.316014E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.361057E-02 | lm-token-buffer-mib: 2.336073E+02 | global_load_balancing_loss: 1.000361E+00 | loss scale: 1.0 | grad norm: 0.118 | params norm: 11196.040 | params rms: 0.060623 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:05:03.490591 | finish at 2026-10-17 17:45:13 + [2026-10-13 14:43:55] iteration 777/ 2384 | consumed samples: 24864 | elapsed time per iteration (ms): 224875.9 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 7.00% | FLOPs estimator: workload-aware | logical tokens/s: 74606.4 | valid tokens/s: 74606.4 | physical tokens/s: 75574.2 | loss tokens/s: 56799.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.837265E-01 | lm-global-supervised-tokens: 1.277290E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.793424E-02 | lm-token-buffer-mib: 2.349457E+02 | global_load_balancing_loss: 9.990916E-01 | loss scale: 1.0 | grad norm: 0.110 | params norm: 11196.004 | params rms: 0.060623 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:22:55.640697 | finish at 2026-10-17 19:06:50 + [2026-10-13 14:47:36] iteration 778/ 2384 | consumed samples: 24896 | elapsed time per iteration (ms): 220824.4 | throughput per GPU (TFLOP/s/GPU): 24.1 | MFU 7.71% | FLOPs estimator: workload-aware | logical tokens/s: 75975.2 | valid tokens/s: 75975.2 | physical tokens/s: 76514.7 | loss tokens/s: 59145.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.083848E-01 | lm-global-supervised-tokens: 1.306071E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.287268E-02 | lm-token-buffer-mib: 2.324814E+02 | global_load_balancing_loss: 9.991760E-01 | loss scale: 1.0 | grad norm: 0.100 | params norm: 11195.967 | params rms: 0.060623 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:30:43.917395 | finish at 2026-10-17 17:18:20 + [2026-10-13 14:51:15] iteration 779/ 2384 | consumed samples: 24928 | elapsed time per iteration (ms): 219198.6 | throughput per GPU (TFLOP/s/GPU): 17.3 | MFU 5.56% | FLOPs estimator: workload-aware | logical tokens/s: 76538.9 | valid tokens/s: 76538.9 | physical tokens/s: 77283.1 | loss tokens/s: 57788.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.221339E-01 | lm-global-supervised-tokens: 1.266722E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.321759E-02 | lm-token-buffer-mib: 2.335692E+02 | global_load_balancing_loss: 9.981020E-01 | loss scale: 1.0 | grad norm: 0.176 | params norm: 11195.930 | params rms: 0.060622 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:43:33.776429 | finish at 2026-10-17 16:34:49 + [2026-10-13 14:55:17] iteration 780/ 2384 | consumed samples: 24960 | elapsed time per iteration (ms): 242114.1 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.73% | FLOPs estimator: workload-aware | logical tokens/s: 69294.6 | valid tokens/s: 69294.6 | physical tokens/s: 69680.9 | loss tokens/s: 52832.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.174286E-01 | lm-global-supervised-tokens: 1.279159E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.796899E-02 | lm-token-buffer-mib: 2.324894E+02 | global_load_balancing_loss: 9.999367E-01 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11195.889 | params rms: 0.060622 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:52:31.026692 | finish at 2026-10-18 02:47:48 + [2026-10-13 14:59:04] iteration 781/ 2384 | consumed samples: 24992 | elapsed time per iteration (ms): 227211.0 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.51% | FLOPs estimator: workload-aware | logical tokens/s: 73839.7 | valid tokens/s: 73839.7 | physical tokens/s: 74420.9 | loss tokens/s: 54654.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.185862E-01 | lm-global-supervised-tokens: 1.241817E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.240866E-02 | lm-token-buffer-mib: 2.342642E+02 | global_load_balancing_loss: 1.001744E+00 | loss scale: 1.0 | grad norm: 0.122 | params norm: 11195.846 | params rms: 0.060622 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:10:19.292566 | finish at 2026-10-17 20:09:23 + [2026-10-13 15:02:49] iteration 782/ 2384 | consumed samples: 25024 | elapsed time per iteration (ms): 225298.7 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.22% | FLOPs estimator: workload-aware | logical tokens/s: 74466.4 | valid tokens/s: 74466.4 | physical tokens/s: 75284.7 | loss tokens/s: 56188.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.101963E-01 | lm-global-supervised-tokens: 1.265920E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.249714E-02 | lm-token-buffer-mib: 2.360652E+02 | global_load_balancing_loss: 1.001254E+00 | loss scale: 1.0 | grad norm: 0.186 | params norm: 11195.804 | params rms: 0.060622 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:15:28.443835 | finish at 2026-10-17 19:18:18 + [2026-10-13 15:06:31] iteration 783/ 2384 | consumed samples: 25056 | elapsed time per iteration (ms): 221464.9 | throughput per GPU (TFLOP/s/GPU): 24.4 | MFU 7.83% | FLOPs estimator: workload-aware | logical tokens/s: 75755.5 | valid tokens/s: 75755.4 | physical tokens/s: 76897.4 | loss tokens/s: 58129.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.451791E-01 | lm-global-supervised-tokens: 1.287355E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.403373E-02 | lm-token-buffer-mib: 2.327535E+02 | global_load_balancing_loss: 1.002255E+00 | loss scale: 1.0 | grad norm: 0.192 | params norm: 11195.764 | params rms: 0.060621 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:29:25.330883 | finish at 2026-10-17 17:35:56 + [2026-10-13 15:10:14] iteration 784/ 2384 | consumed samples: 25088 | elapsed time per iteration (ms): 222743.8 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.25% | FLOPs estimator: workload-aware | logical tokens/s: 75320.5 | valid tokens/s: 75320.5 | physical tokens/s: 76023.8 | loss tokens/s: 53895.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.795786E-01 | lm-global-supervised-tokens: 1.200477E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.302028E-02 | lm-token-buffer-mib: 2.341269E+02 | global_load_balancing_loss: 9.996227E-01 | loss scale: 1.0 | grad norm: 0.306 | params norm: 11195.720 | params rms: 0.060621 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:59:50.073776 | finish at 2026-10-17 18:10:04 + [2026-10-13 15:13:59] iteration 785/ 2384 | consumed samples: 25120 | elapsed time per iteration (ms): 225740.7 | throughput per GPU (TFLOP/s/GPU): 23.9 | MFU 7.65% | FLOPs estimator: workload-aware | logical tokens/s: 74320.7 | valid tokens/s: 74320.7 | physical tokens/s: 74951.3 | loss tokens/s: 58337.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.994999E-01 | lm-global-supervised-tokens: 1.316920E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.443559E-02 | lm-token-buffer-mib: 2.348113E+02 | global_load_balancing_loss: 1.000769E+00 | loss scale: 1.0 | grad norm: 0.191 | params norm: 11195.675 | params rms: 0.060621 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:15:59.341568 | finish at 2026-10-17 19:29:59 + [2026-10-13 15:17:45] iteration 786/ 2384 | consumed samples: 25152 | elapsed time per iteration (ms): 225152.5 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.83% | FLOPs estimator: workload-aware | logical tokens/s: 74514.8 | valid tokens/s: 74514.8 | physical tokens/s: 75187.7 | loss tokens/s: 56717.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.226964E-01 | lm-global-supervised-tokens: 1.276996E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.597932E-02 | lm-token-buffer-mib: 2.346754E+02 | global_load_balancing_loss: 1.001115E+00 | loss scale: 1.0 | grad norm: 0.257 | params norm: 11195.633 | params rms: 0.060621 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:56:33.719627 | finish at 2026-10-17 19:14:18 + [2026-10-13 15:21:29] iteration 787/ 2384 | consumed samples: 25184 | elapsed time per iteration (ms): 224312.7 | throughput per GPU (TFLOP/s/GPU): 25.1 | MFU 8.06% | FLOPs estimator: workload-aware | logical tokens/s: 74793.6 | valid tokens/s: 74793.6 | physical tokens/s: 75424.7 | loss tokens/s: 53623.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.810912E-01 | lm-global-supervised-tokens: 1.202848E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.383741E-02 | lm-token-buffer-mib: 2.343040E+02 | global_load_balancing_loss: 9.992850E-01 | loss scale: 1.0 | grad norm: 0.144 | params norm: 11195.593 | params rms: 0.060620 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:30:27.402133 | finish at 2026-10-17 18:51:56 + [2026-10-13 15:25:15] iteration 788/ 2384 | consumed samples: 25216 | elapsed time per iteration (ms): 226371.4 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.56% | FLOPs estimator: workload-aware | logical tokens/s: 74113.5 | valid tokens/s: 74113.5 | physical tokens/s: 74847.4 | loss tokens/s: 59562.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.282441E-01 | lm-global-supervised-tokens: 1.348321E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.050397E-02 | lm-token-buffer-mib: 2.349936E+02 | global_load_balancing_loss: 9.990958E-01 | loss scale: 1.0 | grad norm: 0.271 | params norm: 11195.553 | params rms: 0.060620 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:21:28.675441 | finish at 2026-10-17 19:46:44 + [2026-10-13 15:28:56] iteration 789/ 2384 | consumed samples: 25248 | elapsed time per iteration (ms): 220940.8 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.36% | FLOPs estimator: workload-aware | logical tokens/s: 75935.2 | valid tokens/s: 75935.2 | physical tokens/s: 76956.4 | loss tokens/s: 56454.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.492123E-01 | lm-global-supervised-tokens: 1.247317E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.472492E-02 | lm-token-buffer-mib: 2.351272E+02 | global_load_balancing_loss: 9.994737E-01 | loss scale: 1.0 | grad norm: 0.330 | params norm: 11195.512 | params rms: 0.060620 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:53:20.551965 | finish at 2026-10-17 17:22:17 + [2026-10-13 15:33:01] iteration 790/ 2384 | consumed samples: 25280 | elapsed time per iteration (ms): 244619.4 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.67% | FLOPs estimator: workload-aware | logical tokens/s: 68584.9 | valid tokens/s: 68584.9 | physical tokens/s: 69014.3 | loss tokens/s: 47133.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.043543E-01 | lm-global-supervised-tokens: 1.152980E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.370977E-02 | lm-token-buffer-mib: 2.342147E+02 | global_load_balancing_loss: 9.998334E-01 | loss scale: 1.0 | grad norm: 0.222 | params norm: 11195.469 | params rms: 0.060620 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:18:43.286775 | finish at 2026-10-18 03:51:44 + [2026-10-13 15:36:47] iteration 791/ 2384 | consumed samples: 25312 | elapsed time per iteration (ms): 226666.3 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.33% | FLOPs estimator: workload-aware | logical tokens/s: 74017.2 | valid tokens/s: 74017.2 | physical tokens/s: 75339.2 | loss tokens/s: 58506.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.370547E-01 | lm-global-supervised-tokens: 1.326133E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.683582E-02 | lm-token-buffer-mib: 2.362511E+02 | global_load_balancing_loss: 1.001021E+00 | loss scale: 1.0 | grad norm: 0.378 | params norm: 11195.427 | params rms: 0.060620 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:17:59.339273 | finish at 2026-10-17 19:54:47 + [2026-10-13 15:40:29] iteration 792/ 2384 | consumed samples: 25344 | elapsed time per iteration (ms): 221500.7 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.03% | FLOPs estimator: workload-aware | logical tokens/s: 75743.1 | valid tokens/s: 75743.1 | physical tokens/s: 76583.9 | loss tokens/s: 55698.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.248490E-01 | lm-global-supervised-tokens: 1.233730E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.371003E-02 | lm-token-buffer-mib: 2.331630E+02 | global_load_balancing_loss: 1.000845E+00 | loss scale: 1.0 | grad norm: 0.106 | params norm: 11195.388 | params rms: 0.060619 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:57:09.128059 | finish at 2026-10-17 17:37:38 + [2026-10-13 15:44:11] iteration 793/ 2384 | consumed samples: 25376 | elapsed time per iteration (ms): 221678.3 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.15% | FLOPs estimator: workload-aware | logical tokens/s: 75682.5 | valid tokens/s: 75682.5 | physical tokens/s: 76177.3 | loss tokens/s: 58430.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.751657E-01 | lm-global-supervised-tokens: 1.295270E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.321280E-02 | lm-token-buffer-mib: 2.337389E+02 | global_load_balancing_loss: 1.001062E+00 | loss scale: 1.0 | grad norm: 0.282 | params norm: 11195.345 | params rms: 0.060619 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:58:10.235080 | finish at 2026-10-17 17:42:21 + [2026-10-13 15:47:51] iteration 794/ 2384 | consumed samples: 25408 | elapsed time per iteration (ms): 219913.1 | throughput per GPU (TFLOP/s/GPU): 19.0 | MFU 6.07% | FLOPs estimator: workload-aware | logical tokens/s: 76290.1 | valid tokens/s: 76290.1 | physical tokens/s: 76768.6 | loss tokens/s: 56295.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.375203E-01 | lm-global-supervised-tokens: 1.238015E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.916904E-02 | lm-token-buffer-mib: 2.327134E+02 | global_load_balancing_loss: 1.002748E+00 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11195.304 | params rms: 0.060619 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:07:41.841516 | finish at 2026-10-17 16:55:32 + [2026-10-13 15:51:36] iteration 795/ 2384 | consumed samples: 25440 | elapsed time per iteration (ms): 225260.4 | throughput per GPU (TFLOP/s/GPU): 24.0 | MFU 7.69% | FLOPs estimator: workload-aware | logical tokens/s: 74478.9 | valid tokens/s: 74478.9 | physical tokens/s: 74792.9 | loss tokens/s: 53077.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.548140E-01 | lm-global-supervised-tokens: 1.195633E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.199105E-02 | lm-token-buffer-mib: 2.329698E+02 | global_load_balancing_loss: 1.001302E+00 | loss scale: 1.0 | grad norm: 0.106 | params norm: 11195.262 | params rms: 0.060619 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:25:38.776448 | finish at 2026-10-17 19:17:15 + [2026-10-13 15:55:16] iteration 796/ 2384 | consumed samples: 25472 | elapsed time per iteration (ms): 219747.5 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.98% | FLOPs estimator: workload-aware | logical tokens/s: 76347.6 | valid tokens/s: 76347.6 | physical tokens/s: 76668.7 | loss tokens/s: 57794.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.083328E-01 | lm-global-supervised-tokens: 1.270028E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.257805E-02 | lm-token-buffer-mib: 2.324185E+02 | global_load_balancing_loss: 9.991595E-01 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11195.215 | params rms: 0.060618 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:55:58.973875 | finish at 2026-10-17 16:51:15 + [2026-10-13 15:58:59] iteration 797/ 2384 | consumed samples: 25504 | elapsed time per iteration (ms): 223842.1 | throughput per GPU (TFLOP/s/GPU): 24.3 | MFU 7.78% | FLOPs estimator: workload-aware | logical tokens/s: 74951.0 | valid tokens/s: 74951.0 | physical tokens/s: 75358.8 | loss tokens/s: 58186.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.864247E-01 | lm-global-supervised-tokens: 1.302468E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.588957E-02 | lm-token-buffer-mib: 2.341437E+02 | global_load_balancing_loss: 9.982485E-01 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11195.170 | params rms: 0.060618 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:40:37.345956 | finish at 2026-10-17 18:39:37 + [2026-10-13 16:02:41] iteration 798/ 2384 | consumed samples: 25536 | elapsed time per iteration (ms): 221185.2 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 7.00% | FLOPs estimator: workload-aware | logical tokens/s: 75851.2 | valid tokens/s: 75851.2 | physical tokens/s: 76666.8 | loss tokens/s: 58685.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.082892E-01 | lm-global-supervised-tokens: 1.298042E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.470509E-02 | lm-token-buffer-mib: 2.338080E+02 | global_load_balancing_loss: 1.000587E+00 | loss scale: 1.0 | grad norm: 0.129 | params norm: 11195.127 | params rms: 0.060618 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:26:39.768000 | finish at 2026-10-17 17:29:20 + [2026-10-13 16:06:19] iteration 799/ 2384 | consumed samples: 25568 | elapsed time per iteration (ms): 218236.5 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.80% | FLOPs estimator: workload-aware | logical tokens/s: 76876.3 | valid tokens/s: 76876.3 | physical tokens/s: 77448.4 | loss tokens/s: 56259.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.864546E-01 | lm-global-supervised-tokens: 1.227780E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.072481E-02 | lm-token-buffer-mib: 2.328205E+02 | global_load_balancing_loss: 1.001183E+00 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11195.087 | params rms: 0.060618 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:05:04.920938 | finish at 2026-10-17 16:11:24 + [2026-10-13 16:10:24] iteration 800/ 2384 | consumed samples: 25600 | elapsed time per iteration (ms): 245691.8 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.71% | FLOPs estimator: workload-aware | logical tokens/s: 68285.5 | valid tokens/s: 68285.5 | physical tokens/s: 68871.7 | loss tokens/s: 51981.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.587285E-01 | lm-global-supervised-tokens: 1.277141E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.237514E-02 | lm-token-buffer-mib: 2.345406E+02 | global_load_balancing_loss: 1.001170E+00 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11195.047 | params rms: 0.060618 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:06:15.860104 | finish at 2026-10-18 04:16:40 +times across ranks (ms): + save-checkpoint: + rank 0: 26201.20 + rank 1: 26201.20 + rank 2: 26201.20 + rank 3: 26201.20 + rank 4: 26201.19 + rank 5: 26201.23 + rank 6: 26201.22 + rank 7: 26201.19 + rank 8: 26200.98 + rank 9: 26201.02 + rank 10: 26201.06 + rank 11: 26200.97 + rank 12: 26201.00 + rank 13: 26200.96 + rank 14: 26201.00 + rank 15: 26200.98 + rank 16: 26202.85 + rank 17: 26202.84 + rank 18: 26202.83 + rank 19: 26202.83 + rank 20: 26202.81 + rank 21: 26202.82 + rank 22: 26202.83 + rank 23: 26202.85 + rank 24: 26201.63 + rank 25: 26201.63 + rank 26: 26201.50 + rank 27: 26201.61 + rank 28: 26201.61 + rank 29: 26201.66 + rank 30: 26201.66 + rank 31: 26201.35 + rank 32: 26201.14 + rank 33: 26201.13 + rank 34: 26201.09 + rank 35: 26201.04 + rank 36: 26201.01 + rank 37: 26200.97 + rank 38: 26201.04 + rank 39: 26200.94 + rank 40: 26201.55 + rank 41: 26201.54 + rank 42: 26201.53 + rank 43: 26201.55 + rank 44: 26201.54 + rank 45: 26201.57 + rank 46: 26201.57 + rank 47: 26201.54 + rank 48: 26201.28 + rank 49: 26201.33 + rank 50: 26201.51 + rank 51: 26201.30 + rank 52: 26201.33 + rank 53: 26201.33 + rank 54: 26201.24 + rank 55: 26201.26 + rank 56: 26201.04 + rank 57: 26201.11 + rank 58: 26201.07 + rank 59: 26201.10 + rank 60: 26201.06 + rank 61: 26201.04 + rank 62: 26201.03 + rank 63: 26201.04 + [2026-10-13 16:14:29] iteration 801/ 2384 | consumed samples: 25632 | elapsed time per iteration (ms): 218002.2 | throughput per GPU (TFLOP/s/GPU): 19.9 | MFU 6.37% | FLOPs estimator: workload-aware | logical tokens/s: 76958.9 | valid tokens/s: 76958.9 | physical tokens/s: 77968.5 | loss tokens/s: 59954.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.054861E-01 | lm-global-supervised-tokens: 1.307018E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.590296E-02 | lm-token-buffer-mib: 2.346473E+02 | global_load_balancing_loss: 1.001118E+00 | loss scale: 1.0 | grad norm: 0.104 | params norm: 11195.007 | params rms: 0.060617 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 23:51:37.428075 | finish at 2026-10-17 16:06:06 + [2026-10-13 16:18:13] iteration 802/ 2384 | consumed samples: 25664 | elapsed time per iteration (ms): 224687.0 | throughput per GPU (TFLOP/s/GPU): 19.4 | MFU 6.23% | FLOPs estimator: workload-aware | logical tokens/s: 74669.1 | valid tokens/s: 74669.1 | physical tokens/s: 75184.8 | loss tokens/s: 55566.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.667674E-01 | lm-global-supervised-tokens: 1.248516E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.327114E-02 | lm-token-buffer-mib: 2.344921E+02 | global_load_balancing_loss: 9.994614E-01 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11194.970 | params rms: 0.060617 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:44:14.835189 | finish at 2026-10-17 19:02:28 + [2026-10-13 16:21:54] iteration 803/ 2384 | consumed samples: 25696 | elapsed time per iteration (ms): 220991.0 | throughput per GPU (TFLOP/s/GPU): 26.2 | MFU 8.39% | FLOPs estimator: workload-aware | logical tokens/s: 75918.1 | valid tokens/s: 75918.0 | physical tokens/s: 76574.9 | loss tokens/s: 56373.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.902967E-01 | lm-global-supervised-tokens: 1.245808E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.941531E-02 | lm-token-buffer-mib: 2.332563E+02 | global_load_balancing_loss: 1.002966E+00 | loss scale: 1.0 | grad norm: 0.103 | params norm: 11194.928 | params rms: 0.060617 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:03:06.737353 | finish at 2026-10-17 17:25:01 + [2026-10-13 16:25:38] iteration 804/ 2384 | consumed samples: 25728 | elapsed time per iteration (ms): 224006.7 | throughput per GPU (TFLOP/s/GPU): 19.6 | MFU 6.27% | FLOPs estimator: workload-aware | logical tokens/s: 74895.9 | valid tokens/s: 74895.9 | physical tokens/s: 76001.7 | loss tokens/s: 56572.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.302764E-01 | lm-global-supervised-tokens: 1.267270E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.960136E-02 | lm-token-buffer-mib: 2.359377E+02 | global_load_balancing_loss: 1.001044E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11194.880 | params rms: 0.060617 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:18:50.650101 | finish at 2026-10-17 18:44:29 + [2026-10-13 16:29:19] iteration 805/ 2384 | consumed samples: 25760 | elapsed time per iteration (ms): 220993.2 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.56% | FLOPs estimator: workload-aware | logical tokens/s: 75917.3 | valid tokens/s: 75917.2 | physical tokens/s: 76682.5 | loss tokens/s: 54403.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.112836E-01 | lm-global-supervised-tokens: 1.202273E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.027527E-02 | lm-token-buffer-mib: 2.334200E+02 | global_load_balancing_loss: 9.990526E-01 | loss scale: 1.0 | grad norm: 0.113 | params norm: 11194.839 | params rms: 0.060616 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:55:48.281347 | finish at 2026-10-17 17:25:08 + [2026-10-13 16:33:02] iteration 806/ 2384 | consumed samples: 25792 | elapsed time per iteration (ms): 222668.1 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.31% | FLOPs estimator: workload-aware | logical tokens/s: 75346.2 | valid tokens/s: 75346.2 | physical tokens/s: 76040.1 | loss tokens/s: 58817.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.792860E-01 | lm-global-supervised-tokens: 1.309688E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.130573E-02 | lm-token-buffer-mib: 2.357880E+02 | global_load_balancing_loss: 1.000564E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11194.801 | params rms: 0.060616 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:36:10.213830 | finish at 2026-10-17 18:09:12 + [2026-10-13 16:36:46] iteration 807/ 2384 | consumed samples: 25824 | elapsed time per iteration (ms): 223513.8 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.34% | FLOPs estimator: workload-aware | logical tokens/s: 75061.1 | valid tokens/s: 75061.1 | physical tokens/s: 76135.8 | loss tokens/s: 49382.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.707956E-01 | lm-global-supervised-tokens: 1.103767E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.813771E-02 | lm-token-buffer-mib: 2.348940E+02 | global_load_balancing_loss: 9.993935E-01 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11194.765 | params rms: 0.060616 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:54:41.313962 | finish at 2026-10-17 18:31:27 + [2026-10-13 16:40:29] iteration 808/ 2384 | consumed samples: 25856 | elapsed time per iteration (ms): 223061.7 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.16% | FLOPs estimator: workload-aware | logical tokens/s: 75213.2 | valid tokens/s: 75213.2 | physical tokens/s: 76068.4 | loss tokens/s: 56633.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.954422E-01 | lm-global-supervised-tokens: 1.263281E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.475216E-02 | lm-token-buffer-mib: 2.339704E+02 | global_load_balancing_loss: 1.000435E+00 | loss scale: 1.0 | grad norm: 0.167 | params norm: 11194.726 | params rms: 0.060616 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:39:05.163090 | finish at 2026-10-17 18:19:34 + [2026-10-13 16:44:13] iteration 809/ 2384 | consumed samples: 25888 | elapsed time per iteration (ms): 224613.0 | throughput per GPU (TFLOP/s/GPU): 18.3 | MFU 5.86% | FLOPs estimator: workload-aware | logical tokens/s: 74693.9 | valid tokens/s: 74693.9 | physical tokens/s: 75288.8 | loss tokens/s: 59528.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.244901E-01 | lm-global-supervised-tokens: 1.337079E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.372572E-02 | lm-token-buffer-mib: 2.337787E+02 | global_load_balancing_loss: 1.000844E+00 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11194.686 | params rms: 0.060616 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:16:05.447080 | finish at 2026-10-17 19:00:19 + [2026-10-13 16:48:20] iteration 810/ 2384 | consumed samples: 25920 | elapsed time per iteration (ms): 247219.7 | throughput per GPU (TFLOP/s/GPU): 19.5 | MFU 6.25% | FLOPs estimator: workload-aware | logical tokens/s: 67863.5 | valid tokens/s: 67863.5 | physical tokens/s: 68997.2 | loss tokens/s: 55968.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.971841E-01 | lm-global-supervised-tokens: 1.383654E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.676104E-02 | lm-token-buffer-mib: 2.347427E+02 | global_load_balancing_loss: 1.000593E+00 | loss scale: 1.0 | grad norm: 0.424 | params norm: 11194.647 | params rms: 0.060615 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:05:23.842104 | finish at 2026-10-18 04:53:44 + [2026-10-13 16:52:01] iteration 811/ 2384 | consumed samples: 25952 | elapsed time per iteration (ms): 220496.4 | throughput per GPU (TFLOP/s/GPU): 26.3 | MFU 8.42% | FLOPs estimator: workload-aware | logical tokens/s: 76088.0 | valid tokens/s: 76088.0 | physical tokens/s: 76380.7 | loss tokens/s: 56881.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.877990E-01 | lm-global-supervised-tokens: 1.254223E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.115925E-02 | lm-token-buffer-mib: 2.331117E+02 | global_load_balancing_loss: 1.001577E+00 | loss scale: 1.0 | grad norm: 0.113 | params norm: 11194.610 | params rms: 0.060615 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:20:40.902641 | finish at 2026-10-17 17:12:42 + [2026-10-13 16:55:45] iteration 812/ 2384 | consumed samples: 25984 | elapsed time per iteration (ms): 223624.7 | throughput per GPU (TFLOP/s/GPU): 23.4 | MFU 7.50% | FLOPs estimator: workload-aware | logical tokens/s: 75023.8 | valid tokens/s: 75023.8 | physical tokens/s: 75579.2 | loss tokens/s: 59207.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.446087E-01 | lm-global-supervised-tokens: 1.324014E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.665939E-02 | lm-token-buffer-mib: 2.341563E+02 | global_load_balancing_loss: 1.000377E+00 | loss scale: 1.0 | grad norm: 0.227 | params norm: 11194.570 | params rms: 0.060615 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:38:58.091849 | finish at 2026-10-17 18:34:43 + [2026-10-13 16:59:32] iteration 813/ 2384 | consumed samples: 26016 | elapsed time per iteration (ms): 227804.8 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.75% | FLOPs estimator: workload-aware | logical tokens/s: 73647.3 | valid tokens/s: 73647.2 | physical tokens/s: 74937.7 | loss tokens/s: 60573.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.105474E-01 | lm-global-supervised-tokens: 1.379897E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.537875E-02 | lm-token-buffer-mib: 2.350952E+02 | global_load_balancing_loss: 9.995657E-01 | loss scale: 1.0 | grad norm: 0.510 | params norm: 11194.528 | params rms: 0.060615 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:24:41.310514 | finish at 2026-10-17 20:24:14 + [2026-10-13 17:03:14] iteration 814/ 2384 | consumed samples: 26048 | elapsed time per iteration (ms): 221679.4 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.55% | FLOPs estimator: workload-aware | logical tokens/s: 75682.3 | valid tokens/s: 75682.3 | physical tokens/s: 76352.4 | loss tokens/s: 57081.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.581076E-01 | lm-global-supervised-tokens: 1.265379E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.868897E-02 | lm-token-buffer-mib: 2.340821E+02 | global_load_balancing_loss: 1.000280E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11194.483 | params rms: 0.060614 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:40:36.619768 | finish at 2026-10-17 17:43:51 + [2026-10-13 17:06:59] iteration 815/ 2384 | consumed samples: 26080 | elapsed time per iteration (ms): 225409.8 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.55% | FLOPs estimator: workload-aware | logical tokens/s: 74429.6 | valid tokens/s: 74429.6 | physical tokens/s: 75063.0 | loss tokens/s: 56539.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.405687E-01 | lm-global-supervised-tokens: 1.274451E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.806890E-02 | lm-token-buffer-mib: 2.344220E+02 | global_load_balancing_loss: 1.000857E+00 | loss scale: 1.0 | grad norm: 0.456 | params norm: 11194.439 | params rms: 0.060614 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:14:27.977031 | finish at 2026-10-17 19:21:27 + [2026-10-13 17:10:42] iteration 816/ 2384 | consumed samples: 26112 | elapsed time per iteration (ms): 222687.8 | throughput per GPU (TFLOP/s/GPU): 24.8 | MFU 7.96% | FLOPs estimator: workload-aware | logical tokens/s: 75339.5 | valid tokens/s: 75339.5 | physical tokens/s: 76060.4 | loss tokens/s: 57465.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.108771E-01 | lm-global-supervised-tokens: 1.279679E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 5.120448E-02 | lm-token-buffer-mib: 2.346417E+02 | global_load_balancing_loss: 9.997471E-01 | loss scale: 1.0 | grad norm: 0.270 | params norm: 11194.394 | params rms: 0.060614 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:59:34.495338 | finish at 2026-10-17 18:10:17 + [2026-10-13 17:14:23] iteration 817/ 2384 | consumed samples: 26144 | elapsed time per iteration (ms): 220992.1 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.38% | FLOPs estimator: workload-aware | logical tokens/s: 75917.6 | valid tokens/s: 75917.6 | physical tokens/s: 76326.4 | loss tokens/s: 55463.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.400319E-01 | lm-global-supervised-tokens: 1.225700E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.518274E-02 | lm-token-buffer-mib: 2.339206E+02 | global_load_balancing_loss: 1.000627E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11194.350 | params rms: 0.060614 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:11:34.597537 | finish at 2026-10-17 17:25:58 + [2026-10-13 17:18:05] iteration 818/ 2384 | consumed samples: 26176 | elapsed time per iteration (ms): 221639.3 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.87% | FLOPs estimator: workload-aware | logical tokens/s: 75696.0 | valid tokens/s: 75696.0 | physical tokens/s: 76258.5 | loss tokens/s: 54958.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.235794E-01 | lm-global-supervised-tokens: 1.218106E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.138640E-02 | lm-token-buffer-mib: 2.330417E+02 | global_load_balancing_loss: 1.000439E+00 | loss scale: 1.0 | grad norm: 0.033 | params norm: 11194.306 | params rms: 0.060614 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:24:47.087965 | finish at 2026-10-17 17:42:52 + [2026-10-13 17:21:52] iteration 819/ 2384 | consumed samples: 26208 | elapsed time per iteration (ms): 227179.9 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.46% | FLOPs estimator: workload-aware | logical tokens/s: 73849.8 | valid tokens/s: 73849.8 | physical tokens/s: 74992.0 | loss tokens/s: 51526.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.181250E-01 | lm-global-supervised-tokens: 1.170580E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.840033E-02 | lm-token-buffer-mib: 2.359761E+02 | global_load_balancing_loss: 1.001973E+00 | loss scale: 1.0 | grad norm: 0.165 | params norm: 11194.263 | params rms: 0.060613 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:45:36.548616 | finish at 2026-10-17 20:07:29 + [2026-10-13 17:25:56] iteration 820/ 2384 | consumed samples: 26240 | elapsed time per iteration (ms): 244461.4 | throughput per GPU (TFLOP/s/GPU): 17.8 | MFU 5.70% | FLOPs estimator: workload-aware | logical tokens/s: 68629.3 | valid tokens/s: 68629.3 | physical tokens/s: 69796.1 | loss tokens/s: 57063.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.399197E-01 | lm-global-supervised-tokens: 1.394975E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.356522E-02 | lm-token-buffer-mib: 2.353237E+02 | global_load_balancing_loss: 9.996774E-01 | loss scale: 1.0 | grad norm: 0.202 | params norm: 11194.219 | params rms: 0.060613 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:12:17.641955 | finish at 2026-10-18 03:38:14 + [2026-10-13 17:29:36] iteration 821/ 2384 | consumed samples: 26272 | elapsed time per iteration (ms): 219235.5 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.63% | FLOPs estimator: workload-aware | logical tokens/s: 76525.9 | valid tokens/s: 76525.9 | physical tokens/s: 77563.8 | loss tokens/s: 50028.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.844905E-01 | lm-global-supervised-tokens: 1.096806E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.135324E-02 | lm-token-buffer-mib: 2.332923E+02 | global_load_balancing_loss: 1.000015E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11194.174 | params rms: 0.060613 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 23:11:05.132115 | finish at 2026-10-17 16:40:41 + [2026-10-13 17:33:20] iteration 822/ 2384 | consumed samples: 26304 | elapsed time per iteration (ms): 224746.8 | throughput per GPU (TFLOP/s/GPU): 25.0 | MFU 8.00% | FLOPs estimator: workload-aware | logical tokens/s: 74649.2 | valid tokens/s: 74649.2 | physical tokens/s: 75132.1 | loss tokens/s: 57109.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.354346E-01 | lm-global-supervised-tokens: 1.283519E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.546447E-02 | lm-token-buffer-mib: 2.340027E+02 | global_load_balancing_loss: 9.991523E-01 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11194.131 | params rms: 0.060613 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:30:54.564080 | finish at 2026-10-17 19:04:15 + [2026-10-13 17:37:09] iteration 823/ 2384 | consumed samples: 26336 | elapsed time per iteration (ms): 228908.0 | throughput per GPU (TFLOP/s/GPU): 18.0 | MFU 5.78% | FLOPs estimator: workload-aware | logical tokens/s: 73292.4 | valid tokens/s: 73292.4 | physical tokens/s: 75222.4 | loss tokens/s: 55378.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.502961E-01 | lm-global-supervised-tokens: 1.267646E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.505246E-02 | lm-token-buffer-mib: 2.382351E+02 | global_load_balancing_loss: 9.995543E-01 | loss scale: 1.0 | grad norm: 0.145 | params norm: 11194.086 | params rms: 0.060612 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:15:25.379056 | finish at 2026-10-17 20:52:35 + [2026-10-13 17:40:50] iteration 824/ 2384 | consumed samples: 26368 | elapsed time per iteration (ms): 220380.0 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.90% | FLOPs estimator: workload-aware | logical tokens/s: 76128.6 | valid tokens/s: 76128.6 | physical tokens/s: 77220.8 | loss tokens/s: 57641.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.749098E-01 | lm-global-supervised-tokens: 1.270306E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.697768E-02 | lm-token-buffer-mib: 2.348090E+02 | global_load_balancing_loss: 1.000511E+00 | loss scale: 1.0 | grad norm: 0.123 | params norm: 11194.041 | params rms: 0.060612 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 23:29:52.861176 | finish at 2026-10-17 17:10:43 + [2026-10-13 17:44:33] iteration 825/ 2384 | consumed samples: 26400 | elapsed time per iteration (ms): 223598.1 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.92% | FLOPs estimator: workload-aware | logical tokens/s: 75032.9 | valid tokens/s: 75032.9 | physical tokens/s: 75952.0 | loss tokens/s: 55462.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.634325E-01 | lm-global-supervised-tokens: 1.240121E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.140697E-02 | lm-token-buffer-mib: 2.348587E+02 | global_load_balancing_loss: 1.001702E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11193.993 | params rms: 0.060612 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:49:49.377603 | finish at 2026-10-17 18:34:23 + [2026-10-13 17:48:19] iteration 826/ 2384 | consumed samples: 26432 | elapsed time per iteration (ms): 226170.7 | throughput per GPU (TFLOP/s/GPU): 24.7 | MFU 7.92% | FLOPs estimator: workload-aware | logical tokens/s: 74179.2 | valid tokens/s: 74179.2 | physical tokens/s: 74844.7 | loss tokens/s: 61750.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.377500E-01 | lm-global-supervised-tokens: 1.396619E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.083885E-02 | lm-token-buffer-mib: 2.350591E+02 | global_load_balancing_loss: 1.002187E+00 | loss scale: 1.0 | grad norm: 0.116 | params norm: 11193.945 | params rms: 0.060612 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:52:53.999746 | finish at 2026-10-17 19:41:13 + [2026-10-13 17:52:05] iteration 827/ 2384 | consumed samples: 26464 | elapsed time per iteration (ms): 225256.5 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.07% | FLOPs estimator: workload-aware | logical tokens/s: 74480.2 | valid tokens/s: 74480.2 | physical tokens/s: 74931.7 | loss tokens/s: 54676.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.652484E-01 | lm-global-supervised-tokens: 1.231633E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.221072E-02 | lm-token-buffer-mib: 2.331365E+02 | global_load_balancing_loss: 1.001146E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11193.896 | params rms: 0.060611 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:25:24.413199 | finish at 2026-10-17 19:17:29 + [2026-10-13 17:55:47] iteration 828/ 2384 | consumed samples: 26496 | elapsed time per iteration (ms): 222216.4 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.42% | FLOPs estimator: workload-aware | logical tokens/s: 75499.2 | valid tokens/s: 75499.2 | physical tokens/s: 76065.1 | loss tokens/s: 55322.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.254675E-01 | lm-global-supervised-tokens: 1.229364E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.368602E-02 | lm-token-buffer-mib: 2.331855E+02 | global_load_balancing_loss: 1.001053E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11193.846 | params rms: 0.060611 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:02:48.746822 | finish at 2026-10-17 17:58:36 + [2026-10-13 17:59:30] iteration 829/ 2384 | consumed samples: 26528 | elapsed time per iteration (ms): 222700.1 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.59% | FLOPs estimator: workload-aware | logical tokens/s: 75335.4 | valid tokens/s: 75335.4 | physical tokens/s: 75480.3 | loss tokens/s: 56709.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.647353E-01 | lm-global-supervised-tokens: 1.262922E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.224970E-02 | lm-token-buffer-mib: 2.323230E+02 | global_load_balancing_loss: 1.000517E+00 | loss scale: 1.0 | grad norm: 0.180 | params norm: 11193.799 | params rms: 0.060611 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:11:38.599062 | finish at 2026-10-17 18:11:08 + [2026-10-13 18:03:33] iteration 830/ 2384 | consumed samples: 26560 | elapsed time per iteration (ms): 243271.8 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.58% | FLOPs estimator: workload-aware | logical tokens/s: 68964.7 | valid tokens/s: 68964.7 | physical tokens/s: 69797.0 | loss tokens/s: 52539.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.406897E-01 | lm-global-supervised-tokens: 1.278137E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.367497E-02 | lm-token-buffer-mib: 2.333785E+02 | global_load_balancing_loss: 1.001721E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11193.754 | params rms: 0.060611 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:00:44.337250 | finish at 2026-10-18 03:04:17 + [2026-10-13 18:07:16] iteration 831/ 2384 | consumed samples: 26592 | elapsed time per iteration (ms): 223019.5 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.81% | FLOPs estimator: workload-aware | logical tokens/s: 75227.5 | valid tokens/s: 75227.5 | physical tokens/s: 76020.5 | loss tokens/s: 60073.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.791529E-01 | lm-global-supervised-tokens: 1.339744E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.507560E-02 | lm-token-buffer-mib: 2.341376E+02 | global_load_balancing_loss: 1.001026E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11193.709 | params rms: 0.060610 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:12:29.273159 | finish at 2026-10-17 18:19:45 + [2026-10-13 18:10:59] iteration 832/ 2384 | consumed samples: 26624 | elapsed time per iteration (ms): 223109.2 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.01% | FLOPs estimator: workload-aware | logical tokens/s: 75197.3 | valid tokens/s: 75197.3 | physical tokens/s: 76054.2 | loss tokens/s: 58664.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.678255E-01 | lm-global-supervised-tokens: 1.308849E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.541106E-02 | lm-token-buffer-mib: 2.357405E+02 | global_load_balancing_loss: 9.992364E-01 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11193.667 | params rms: 0.060610 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:11:05.479881 | finish at 2026-10-17 18:22:05 + [2026-10-13 18:14:41] iteration 833/ 2384 | consumed samples: 26656 | elapsed time per iteration (ms): 221611.3 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.21% | FLOPs estimator: workload-aware | logical tokens/s: 75705.5 | valid tokens/s: 75705.5 | physical tokens/s: 76367.2 | loss tokens/s: 56278.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.766150E-01 | lm-global-supervised-tokens: 1.247200E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.164368E-02 | lm-token-buffer-mib: 2.349435E+02 | global_load_balancing_loss: 9.986175E-01 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11193.629 | params rms: 0.060610 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 23:28:39.068230 | finish at 2026-10-17 17:43:20 + [2026-10-13 18:18:25] iteration 834/ 2384 | consumed samples: 26688 | elapsed time per iteration (ms): 224328.5 | throughput per GPU (TFLOP/s/GPU): 25.3 | MFU 8.11% | FLOPs estimator: workload-aware | logical tokens/s: 74788.5 | valid tokens/s: 74788.5 | physical tokens/s: 75299.9 | loss tokens/s: 54673.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.447333E-01 | lm-global-supervised-tokens: 1.226471E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.790277E-02 | lm-token-buffer-mib: 2.336169E+02 | global_load_balancing_loss: 9.994605E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11193.593 | params rms: 0.060610 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:35:09.161377 | finish at 2026-10-17 18:53:34 + [2026-10-13 18:22:10] iteration 835/ 2384 | consumed samples: 26720 | elapsed time per iteration (ms): 224660.0 | throughput per GPU (TFLOP/s/GPU): 24.8 | MFU 7.95% | FLOPs estimator: workload-aware | logical tokens/s: 74678.2 | valid tokens/s: 74678.2 | physical tokens/s: 75424.1 | loss tokens/s: 59543.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.359368E-01 | lm-global-supervised-tokens: 1.337693E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.335631E-02 | lm-token-buffer-mib: 2.350654E+02 | global_load_balancing_loss: 9.985675E-01 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11193.557 | params rms: 0.060609 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:39:58.272180 | finish at 2026-10-17 19:02:08 + [2026-10-13 18:25:51] iteration 836/ 2384 | consumed samples: 26752 | elapsed time per iteration (ms): 220942.8 | throughput per GPU (TFLOP/s/GPU): 18.7 | MFU 5.99% | FLOPs estimator: workload-aware | logical tokens/s: 75934.6 | valid tokens/s: 75934.5 | physical tokens/s: 76814.7 | loss tokens/s: 60000.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.811486E-01 | lm-global-supervised-tokens: 1.325673E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.410581E-02 | lm-token-buffer-mib: 2.344851E+02 | global_load_balancing_loss: 1.002022E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11193.519 | params rms: 0.060609 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 23:00:19.512783 | finish at 2026-10-17 17:26:10 + [2026-10-13 18:29:34] iteration 837/ 2384 | consumed samples: 26784 | elapsed time per iteration (ms): 223560.9 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.13% | FLOPs estimator: workload-aware | logical tokens/s: 75045.2 | valid tokens/s: 75045.2 | physical tokens/s: 75607.9 | loss tokens/s: 54597.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.741410E-01 | lm-global-supervised-tokens: 1.220595E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.672908E-02 | lm-token-buffer-mib: 2.330737E+02 | global_load_balancing_loss: 1.001037E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11193.478 | params rms: 0.060609 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:04:08.663203 | finish at 2026-10-17 18:33:43 + [2026-10-13 18:33:15] iteration 838/ 2384 | consumed samples: 26816 | elapsed time per iteration (ms): 220949.4 | throughput per GPU (TFLOP/s/GPU): 16.9 | MFU 5.43% | FLOPs estimator: workload-aware | logical tokens/s: 75932.3 | valid tokens/s: 75932.3 | physical tokens/s: 76687.3 | loss tokens/s: 58874.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.096476E-01 | lm-global-supervised-tokens: 1.300825E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.311281E-02 | lm-token-buffer-mib: 2.340762E+02 | global_load_balancing_loss: 1.001466E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11193.438 | params rms: 0.060609 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:53:07.733986 | finish at 2026-10-17 17:26:23 + [2026-10-13 18:36:58] iteration 839/ 2384 | consumed samples: 26848 | elapsed time per iteration (ms): 222445.1 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.07% | FLOPs estimator: workload-aware | logical tokens/s: 75421.7 | valid tokens/s: 75421.7 | physical tokens/s: 75897.4 | loss tokens/s: 58465.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.124433E-01 | lm-global-supervised-tokens: 1.300545E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.257164E-02 | lm-token-buffer-mib: 2.330382E+02 | global_load_balancing_loss: 1.000996E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11193.399 | params rms: 0.060609 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 23:27:57.668924 | finish at 2026-10-17 18:04:55 + [2026-10-13 18:41:08] iteration 840/ 2384 | consumed samples: 26880 | elapsed time per iteration (ms): 250615.5 | throughput per GPU (TFLOP/s/GPU): 24.0 | MFU 7.71% | FLOPs estimator: workload-aware | logical tokens/s: 66944.0 | valid tokens/s: 66944.0 | physical tokens/s: 67617.2 | loss tokens/s: 51401.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.876557E-01 | lm-global-supervised-tokens: 1.288208E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.420423E-02 | lm-token-buffer-mib: 2.364014E+02 | global_load_balancing_loss: 1.002533E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11193.359 | params rms: 0.060608 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:29:10.366930 | finish at 2026-10-18 06:10:19 +times across ranks (ms): + save-checkpoint: + rank 0: 24829.54 + rank 1: 24829.53 + rank 2: 24829.54 + rank 3: 24829.54 + rank 4: 24829.53 + rank 5: 24829.54 + rank 6: 24829.55 + rank 7: 24829.51 + rank 8: 24830.38 + rank 9: 24830.42 + rank 10: 24830.43 + rank 11: 24830.38 + rank 12: 24830.41 + rank 13: 24830.38 + rank 14: 24830.39 + rank 15: 24830.38 + rank 16: 24830.71 + rank 17: 24830.69 + rank 18: 24830.71 + rank 19: 24830.74 + rank 20: 24830.68 + rank 21: 24830.67 + rank 22: 24830.70 + rank 23: 24830.73 + rank 24: 24830.39 + rank 25: 24830.40 + rank 26: 24830.36 + rank 27: 24830.29 + rank 28: 24830.33 + rank 29: 24830.32 + rank 30: 24830.28 + rank 31: 24830.41 + rank 32: 24830.64 + rank 33: 24830.84 + rank 34: 24830.68 + rank 35: 24830.66 + rank 36: 24830.74 + rank 37: 24830.65 + rank 38: 24830.64 + rank 39: 24830.75 + rank 40: 24830.55 + rank 41: 24830.60 + rank 42: 24830.54 + rank 43: 24830.56 + rank 44: 24830.59 + rank 45: 24830.57 + rank 46: 24830.54 + rank 47: 24830.54 + rank 48: 24830.26 + rank 49: 24830.20 + rank 50: 24830.35 + rank 51: 24830.29 + rank 52: 24830.27 + rank 53: 24830.14 + rank 54: 24830.14 + rank 55: 24830.08 + rank 56: 24830.44 + rank 57: 24830.46 + rank 58: 24830.46 + rank 59: 24830.30 + rank 60: 24830.47 + rank 61: 24830.45 + rank 62: 24830.43 + rank 63: 24830.44 + [2026-10-13 18:45:29] iteration 841/ 2384 | consumed samples: 26912 | elapsed time per iteration (ms): 235636.7 | throughput per GPU (TFLOP/s/GPU): 26.1 | MFU 8.37% | FLOPs estimator: workload-aware | logical tokens/s: 71199.3 | valid tokens/s: 71199.3 | physical tokens/s: 71913.5 | loss tokens/s: 56249.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.226133E-01 | lm-global-supervised-tokens: 1.325433E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 6.493320E-01 | lm-token-buffer-mib: 2.330402E+02 | global_load_balancing_loss: 1.002613E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11193.319 | params rms: 0.060608 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:59:47.485726 | finish at 2026-10-17 23:45:16 + failed to log checkpoint /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/iter_0000840 as wandb artifact: Unable to connect to server to verify API token. + [2026-10-13 18:49:20] iteration 842/ 2384 | consumed samples: 26944 | elapsed time per iteration (ms): 231442.8 | throughput per GPU (TFLOP/s/GPU): 16.9 | MFU 5.43% | FLOPs estimator: workload-aware | logical tokens/s: 72489.6 | valid tokens/s: 72489.6 | physical tokens/s: 73466.6 | loss tokens/s: 56944.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.417447E-01 | lm-global-supervised-tokens: 1.317940E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.223036E-02 | lm-token-buffer-mib: 2.338121E+02 | global_load_balancing_loss: 9.993495E-01 | loss scale: 1.0 | grad norm: 0.138 | params norm: 11193.277 | params rms: 0.060608 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:08:04.842889 | finish at 2026-10-17 21:57:25 + [2026-10-13 18:53:03] iteration 843/ 2384 | consumed samples: 26976 | elapsed time per iteration (ms): 222586.7 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.76% | FLOPs estimator: workload-aware | logical tokens/s: 75373.7 | valid tokens/s: 75373.7 | physical tokens/s: 76875.3 | loss tokens/s: 59864.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.294487E-01 | lm-global-supervised-tokens: 1.332503E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.366638E-02 | lm-token-buffer-mib: 2.350647E+02 | global_load_balancing_loss: 9.992112E-01 | loss scale: 1.0 | grad norm: 0.148 | params norm: 11193.234 | params rms: 0.060608 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 23:16:46.067902 | finish at 2026-10-17 18:09:49 + [2026-10-13 18:56:44] iteration 844/ 2384 | consumed samples: 27008 | elapsed time per iteration (ms): 221288.3 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.03% | FLOPs estimator: workload-aware | logical tokens/s: 75816.0 | valid tokens/s: 75816.0 | physical tokens/s: 76571.0 | loss tokens/s: 58199.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.916058E-01 | lm-global-supervised-tokens: 1.287877E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.327050E-02 | lm-token-buffer-mib: 2.337727E+02 | global_load_balancing_loss: 9.983931E-01 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11193.193 | params rms: 0.060607 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:39:43.967738 | finish at 2026-10-17 17:36:28 + [2026-10-13 19:00:25] iteration 845/ 2384 | consumed samples: 27040 | elapsed time per iteration (ms): 221424.2 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.69% | FLOPs estimator: workload-aware | logical tokens/s: 75769.5 | valid tokens/s: 75769.5 | physical tokens/s: 76370.5 | loss tokens/s: 59470.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.973121E-01 | lm-global-supervised-tokens: 1.316819E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.515691E-02 | lm-token-buffer-mib: 2.328472E+02 | global_load_balancing_loss: 9.985907E-01 | loss scale: 1.0 | grad norm: 0.127 | params norm: 11193.152 | params rms: 0.060607 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:39:31.845357 | finish at 2026-10-17 17:39:57 + [2026-10-13 19:04:07] iteration 846/ 2384 | consumed samples: 27072 | elapsed time per iteration (ms): 221268.0 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.10% | FLOPs estimator: workload-aware | logical tokens/s: 75823.0 | valid tokens/s: 75823.0 | physical tokens/s: 76064.0 | loss tokens/s: 57700.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.696271E-01 | lm-global-supervised-tokens: 1.276731E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.256251E-02 | lm-token-buffer-mib: 2.327261E+02 | global_load_balancing_loss: 1.000217E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11193.110 | params rms: 0.060607 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:31:50.213162 | finish at 2026-10-17 17:35:57 + [2026-10-13 19:07:52] iteration 847/ 2384 | consumed samples: 27104 | elapsed time per iteration (ms): 225012.6 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.35% | FLOPs estimator: workload-aware | logical tokens/s: 74561.1 | valid tokens/s: 74561.1 | physical tokens/s: 76079.2 | loss tokens/s: 60430.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.804529E-01 | lm-global-supervised-tokens: 1.359768E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.593379E-02 | lm-token-buffer-mib: 2.395210E+02 | global_load_balancing_loss: 1.000044E+00 | loss scale: 1.0 | grad norm: 0.154 | params norm: 11193.068 | params rms: 0.060607 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:04:04.298322 | finish at 2026-10-17 19:11:56 + [2026-10-13 19:11:31] iteration 848/ 2384 | consumed samples: 27136 | elapsed time per iteration (ms): 219212.6 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.79% | FLOPs estimator: workload-aware | logical tokens/s: 76533.9 | valid tokens/s: 76533.9 | physical tokens/s: 77347.1 | loss tokens/s: 58864.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.976988E-01 | lm-global-supervised-tokens: 1.290380E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.826616E-02 | lm-token-buffer-mib: 2.330449E+02 | global_load_balancing_loss: 1.001069E+00 | loss scale: 1.0 | grad norm: 0.133 | params norm: 11193.025 | params rms: 0.060607 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:31:50.595337 | finish at 2026-10-17 16:43:22 + [2026-10-13 19:15:13] iteration 849/ 2384 | consumed samples: 27168 | elapsed time per iteration (ms): 221652.4 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.78% | FLOPs estimator: workload-aware | logical tokens/s: 75691.5 | valid tokens/s: 75691.5 | physical tokens/s: 76578.3 | loss tokens/s: 59820.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.330699E-01 | lm-global-supervised-tokens: 1.325930E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.870280E-02 | lm-token-buffer-mib: 2.328116E+02 | global_load_balancing_loss: 1.002499E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11192.980 | params rms: 0.060606 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:30:36.425608 | finish at 2026-10-17 17:45:49 + [2026-10-13 19:19:16] iteration 850/ 2384 | consumed samples: 27200 | elapsed time per iteration (ms): 243935.6 | throughput per GPU (TFLOP/s/GPU): 18.9 | MFU 6.07% | FLOPs estimator: workload-aware | logical tokens/s: 68777.2 | valid tokens/s: 68777.2 | physical tokens/s: 69769.5 | loss tokens/s: 53680.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.337282E-01 | lm-global-supervised-tokens: 1.309462E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.187716E-02 | lm-token-buffer-mib: 2.336841E+02 | global_load_balancing_loss: 1.001484E+00 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11192.933 | params rms: 0.060606 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:56:37.180475 | finish at 2026-10-18 03:15:54 + [2026-10-13 19:23:04] iteration 851/ 2384 | consumed samples: 27232 | elapsed time per iteration (ms): 227377.2 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.03% | FLOPs estimator: workload-aware | logical tokens/s: 73785.7 | valid tokens/s: 73785.7 | physical tokens/s: 74506.4 | loss tokens/s: 57450.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.666534E-01 | lm-global-supervised-tokens: 1.306283E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.444799E-02 | lm-token-buffer-mib: 2.344109E+02 | global_load_balancing_loss: 1.001802E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11192.882 | params rms: 0.060606 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:49:29.308101 | finish at 2026-10-17 20:12:33 + [2026-10-13 19:26:47] iteration 852/ 2384 | consumed samples: 27264 | elapsed time per iteration (ms): 222932.2 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.76% | FLOPs estimator: workload-aware | logical tokens/s: 75257.0 | valid tokens/s: 75257.0 | physical tokens/s: 76080.1 | loss tokens/s: 58968.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.212005E-01 | lm-global-supervised-tokens: 1.314603E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.388327E-02 | lm-token-buffer-mib: 2.345519E+02 | global_load_balancing_loss: 1.000620E+00 | loss scale: 1.0 | grad norm: 0.274 | params norm: 11192.832 | params rms: 0.060606 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:52:12.114990 | finish at 2026-10-17 18:18:59 + [2026-10-13 19:30:33] iteration 853/ 2384 | consumed samples: 27296 | elapsed time per iteration (ms): 225844.9 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.84% | FLOPs estimator: workload-aware | logical tokens/s: 74286.4 | valid tokens/s: 74286.4 | physical tokens/s: 74798.8 | loss tokens/s: 53909.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.575367E-01 | lm-global-supervised-tokens: 1.217510E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.264137E-02 | lm-token-buffer-mib: 2.338782E+02 | global_load_balancing_loss: 1.000573E+00 | loss scale: 1.0 | grad norm: 0.127 | params norm: 11192.782 | params rms: 0.060605 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:02:48.508884 | finish at 2026-10-17 19:33:21 + [2026-10-13 19:34:16] iteration 854/ 2384 | consumed samples: 27328 | elapsed time per iteration (ms): 222958.2 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.36% | FLOPs estimator: workload-aware | logical tokens/s: 75248.2 | valid tokens/s: 75248.2 | physical tokens/s: 75787.3 | loss tokens/s: 59276.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.002176E-01 | lm-global-supervised-tokens: 1.321618E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.636110E-02 | lm-token-buffer-mib: 2.331548E+02 | global_load_balancing_loss: 9.999468E-01 | loss scale: 1.0 | grad norm: 0.289 | params norm: 11192.735 | params rms: 0.060605 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:45:25.998180 | finish at 2026-10-17 18:19:42 + [2026-10-13 19:37:55] iteration 855/ 2384 | consumed samples: 27360 | elapsed time per iteration (ms): 219418.5 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.40% | FLOPs estimator: workload-aware | logical tokens/s: 76462.1 | valid tokens/s: 76462.1 | physical tokens/s: 77054.3 | loss tokens/s: 59781.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.012111E-01 | lm-global-supervised-tokens: 1.311726E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.627972E-02 | lm-token-buffer-mib: 2.347130E+02 | global_load_balancing_loss: 9.991597E-01 | loss scale: 1.0 | grad norm: 0.277 | params norm: 11192.687 | params rms: 0.060605 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:11:30.848506 | finish at 2026-10-17 16:49:26 + [2026-10-13 19:41:39] iteration 856/ 2384 | consumed samples: 27392 | elapsed time per iteration (ms): 224051.1 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.34% | FLOPs estimator: workload-aware | logical tokens/s: 74881.2 | valid tokens/s: 74881.2 | physical tokens/s: 75413.1 | loss tokens/s: 58093.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.553335E-01 | lm-global-supervised-tokens: 1.301589E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.166077E-02 | lm-token-buffer-mib: 2.342206E+02 | global_load_balancing_loss: 1.000132E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11192.639 | params rms: 0.060605 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 23:05:50.041115 | finish at 2026-10-17 18:47:29 + [2026-10-13 19:45:21] iteration 857/ 2384 | consumed samples: 27424 | elapsed time per iteration (ms): 222349.4 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.40% | FLOPs estimator: workload-aware | logical tokens/s: 75454.2 | valid tokens/s: 75454.2 | physical tokens/s: 75684.0 | loss tokens/s: 55498.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.852357E-01 | lm-global-supervised-tokens: 1.233994E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.600938E-02 | lm-token-buffer-mib: 2.327923E+02 | global_load_balancing_loss: 1.000494E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11192.592 | params rms: 0.060604 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:18:47.491271 | finish at 2026-10-17 18:04:09 + [2026-10-13 19:49:01] iteration 858/ 2384 | consumed samples: 27456 | elapsed time per iteration (ms): 219614.1 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.44% | FLOPs estimator: workload-aware | logical tokens/s: 76394.0 | valid tokens/s: 76394.0 | physical tokens/s: 76944.9 | loss tokens/s: 58813.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.464020E-01 | lm-global-supervised-tokens: 1.291634E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.908604E-02 | lm-token-buffer-mib: 2.333309E+02 | global_load_balancing_loss: 1.000573E+00 | loss scale: 1.0 | grad norm: 0.206 | params norm: 11192.549 | params rms: 0.060604 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:05:31.145675 | finish at 2026-10-17 16:54:32 + [2026-10-13 19:52:43] iteration 859/ 2384 | consumed samples: 27488 | elapsed time per iteration (ms): 222285.6 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.22% | FLOPs estimator: workload-aware | logical tokens/s: 75475.9 | valid tokens/s: 75475.9 | physical tokens/s: 76245.2 | loss tokens/s: 58001.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.442196E-01 | lm-global-supervised-tokens: 1.289298E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.299152E-02 | lm-token-buffer-mib: 2.350679E+02 | global_load_balancing_loss: 1.000430E+00 | loss scale: 1.0 | grad norm: 0.117 | params norm: 11192.512 | params rms: 0.060604 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:09:45.611182 | finish at 2026-10-17 18:02:29 + [2026-10-13 19:56:46] iteration 860/ 2384 | consumed samples: 27520 | elapsed time per iteration (ms): 243148.2 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.99% | FLOPs estimator: workload-aware | logical tokens/s: 68999.8 | valid tokens/s: 68999.8 | physical tokens/s: 69566.1 | loss tokens/s: 52627.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.373590E-01 | lm-global-supervised-tokens: 1.279634E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.555645E-02 | lm-token-buffer-mib: 2.332253E+02 | global_load_balancing_loss: 9.998353E-01 | loss scale: 1.0 | grad norm: 0.190 | params norm: 11192.477 | params rms: 0.060604 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:55:57.925527 | finish at 2026-10-18 02:52:44 + [2026-10-13 20:00:27] iteration 861/ 2384 | consumed samples: 27552 | elapsed time per iteration (ms): 220684.4 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.84% | FLOPs estimator: workload-aware | logical tokens/s: 76023.5 | valid tokens/s: 76023.5 | physical tokens/s: 77191.7 | loss tokens/s: 57885.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.887533E-01 | lm-global-supervised-tokens: 1.277434E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.777363E-02 | lm-token-buffer-mib: 2.339445E+02 | global_load_balancing_loss: 1.001099E+00 | loss scale: 1.0 | grad norm: 0.154 | params norm: 11192.434 | params rms: 0.060603 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:21:42.306466 | finish at 2026-10-17 17:22:09 + [2026-10-13 20:04:07] iteration 862/ 2384 | consumed samples: 27584 | elapsed time per iteration (ms): 219834.0 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.72% | FLOPs estimator: workload-aware | logical tokens/s: 76317.5 | valid tokens/s: 76317.5 | physical tokens/s: 77081.0 | loss tokens/s: 56721.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.422428E-01 | lm-global-supervised-tokens: 1.246939E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.683330E-02 | lm-token-buffer-mib: 2.335298E+02 | global_load_balancing_loss: 1.000329E+00 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11192.390 | params rms: 0.060603 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:56:27.303535 | finish at 2026-10-17 17:00:34 + [2026-10-13 20:07:48] iteration 863/ 2384 | consumed samples: 27616 | elapsed time per iteration (ms): 221236.1 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.73% | FLOPs estimator: workload-aware | logical tokens/s: 75833.8 | valid tokens/s: 75833.8 | physical tokens/s: 76482.9 | loss tokens/s: 59192.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.451305E-01 | lm-global-supervised-tokens: 1.309553E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.327098E-02 | lm-token-buffer-mib: 2.332395E+02 | global_load_balancing_loss: 9.998851E-01 | loss scale: 1.0 | grad norm: 0.109 | params norm: 11192.349 | params rms: 0.060603 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:28:20.081927 | finish at 2026-10-17 17:36:08 + [2026-10-13 20:11:29] iteration 864/ 2384 | consumed samples: 27648 | elapsed time per iteration (ms): 221170.9 | throughput per GPU (TFLOP/s/GPU): 19.5 | MFU 6.25% | FLOPs estimator: workload-aware | logical tokens/s: 75856.3 | valid tokens/s: 75856.3 | physical tokens/s: 76418.6 | loss tokens/s: 60468.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.513175E-01 | lm-global-supervised-tokens: 1.337388E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.623905E-02 | lm-token-buffer-mib: 2.336843E+02 | global_load_balancing_loss: 1.001273E+00 | loss scale: 1.0 | grad norm: 0.197 | params norm: 11192.308 | params rms: 0.060603 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:22:59.704380 | finish at 2026-10-17 17:34:29 + [2026-10-13 20:15:12] iteration 865/ 2384 | consumed samples: 27680 | elapsed time per iteration (ms): 222591.4 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.56% | FLOPs estimator: workload-aware | logical tokens/s: 75372.1 | valid tokens/s: 75372.1 | physical tokens/s: 76205.8 | loss tokens/s: 61224.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.825656E-01 | lm-global-supervised-tokens: 1.362803E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.796472E-02 | lm-token-buffer-mib: 2.336288E+02 | global_load_balancing_loss: 1.001817E+00 | loss scale: 1.0 | grad norm: 0.175 | params norm: 11192.266 | params rms: 0.060602 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:55:16.377384 | finish at 2026-10-17 18:10:28 + [2026-10-13 20:18:57] iteration 866/ 2384 | consumed samples: 27712 | elapsed time per iteration (ms): 225105.6 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.84% | FLOPs estimator: workload-aware | logical tokens/s: 74530.4 | valid tokens/s: 74530.4 | physical tokens/s: 76055.8 | loss tokens/s: 57915.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.557159E-01 | lm-global-supervised-tokens: 1.303700E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.182775E-02 | lm-token-buffer-mib: 2.368633E+02 | global_load_balancing_loss: 1.000969E+00 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11192.220 | params rms: 0.060602 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:55:10.268407 | finish at 2026-10-17 19:14:07 + [2026-10-13 20:22:43] iteration 867/ 2384 | consumed samples: 27744 | elapsed time per iteration (ms): 226293.3 | throughput per GPU (TFLOP/s/GPU): 19.3 | MFU 6.18% | FLOPs estimator: workload-aware | logical tokens/s: 74139.2 | valid tokens/s: 74139.2 | physical tokens/s: 74640.1 | loss tokens/s: 57321.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.880495E-01 | lm-global-supervised-tokens: 1.297157E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.787601E-02 | lm-token-buffer-mib: 2.326698E+02 | global_load_balancing_loss: 1.001303E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11192.175 | params rms: 0.060602 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 23:21:26.955138 | finish at 2026-10-17 19:44:10 + [2026-10-13 20:26:28] iteration 868/ 2384 | consumed samples: 27776 | elapsed time per iteration (ms): 224318.4 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.22% | FLOPs estimator: workload-aware | logical tokens/s: 74791.9 | valid tokens/s: 74791.9 | physical tokens/s: 75644.2 | loss tokens/s: 58187.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.715529E-01 | lm-global-supervised-tokens: 1.305252E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.247917E-02 | lm-token-buffer-mib: 2.346296E+02 | global_load_balancing_loss: 1.000471E+00 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11192.132 | params rms: 0.060602 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:27:46.721727 | finish at 2026-10-17 18:54:14 + [2026-10-13 20:30:08] iteration 869/ 2384 | consumed samples: 27808 | elapsed time per iteration (ms): 219849.6 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.16% | FLOPs estimator: workload-aware | logical tokens/s: 76312.1 | valid tokens/s: 76312.1 | physical tokens/s: 76946.3 | loss tokens/s: 62602.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.311321E-01 | lm-global-supervised-tokens: 1.376320E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.262559E-02 | lm-token-buffer-mib: 2.336524E+02 | global_load_balancing_loss: 9.987952E-01 | loss scale: 1.0 | grad norm: 0.146 | params norm: 11192.091 | params rms: 0.060602 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:31:12.171929 | finish at 2026-10-17 17:01:20 + [2026-10-13 20:34:11] iteration 870/ 2384 | consumed samples: 27840 | elapsed time per iteration (ms): 243120.4 | throughput per GPU (TFLOP/s/GPU): 18.0 | MFU 5.77% | FLOPs estimator: workload-aware | logical tokens/s: 69007.8 | valid tokens/s: 69007.8 | physical tokens/s: 69546.2 | loss tokens/s: 53002.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.854300E-01 | lm-global-supervised-tokens: 1.288589E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.786050E-02 | lm-token-buffer-mib: 2.335450E+02 | global_load_balancing_loss: 9.992610E-01 | loss scale: 1.0 | grad norm: 0.154 | params norm: 11192.052 | params rms: 0.060601 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:14:44.211168 | finish at 2026-10-18 02:48:55 + [2026-10-13 20:37:53] iteration 871/ 2384 | consumed samples: 27872 | elapsed time per iteration (ms): 222716.7 | throughput per GPU (TFLOP/s/GPU): 23.5 | MFU 7.52% | FLOPs estimator: workload-aware | logical tokens/s: 75329.7 | valid tokens/s: 75329.7 | physical tokens/s: 75787.9 | loss tokens/s: 51572.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.635157E-01 | lm-global-supervised-tokens: 1.148602E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.735165E-02 | lm-token-buffer-mib: 2.339758E+02 | global_load_balancing_loss: 1.000707E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11192.011 | params rms: 0.060601 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:36:10.381646 | finish at 2026-10-17 18:14:04 + [2026-10-13 20:41:38] iteration 872/ 2384 | consumed samples: 27904 | elapsed time per iteration (ms): 225040.7 | throughput per GPU (TFLOP/s/GPU): 17.2 | MFU 5.52% | FLOPs estimator: workload-aware | logical tokens/s: 74551.8 | valid tokens/s: 74551.8 | physical tokens/s: 76348.4 | loss tokens/s: 55022.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.919465E-01 | lm-global-supervised-tokens: 1.238228E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.630390E-02 | lm-token-buffer-mib: 2.354722E+02 | global_load_balancing_loss: 9.994236E-01 | loss scale: 1.0 | grad norm: 0.391 | params norm: 11191.967 | params rms: 0.060601 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:31:01.593853 | finish at 2026-10-17 19:12:40 + [2026-10-13 20:45:18] iteration 873/ 2384 | consumed samples: 27936 | elapsed time per iteration (ms): 219566.7 | throughput per GPU (TFLOP/s/GPU): 20.2 | MFU 6.49% | FLOPs estimator: workload-aware | logical tokens/s: 76410.5 | valid tokens/s: 76410.5 | physical tokens/s: 77097.0 | loss tokens/s: 58085.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.400709E-01 | lm-global-supervised-tokens: 1.275370E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.483779E-02 | lm-token-buffer-mib: 2.337541E+02 | global_load_balancing_loss: 1.000025E+00 | loss scale: 1.0 | grad norm: 0.095 | params norm: 11191.923 | params rms: 0.060601 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:09:25.261697 | finish at 2026-10-17 16:54:43 + [2026-10-13 20:49:03] iteration 874/ 2384 | consumed samples: 27968 | elapsed time per iteration (ms): 225002.8 | throughput per GPU (TFLOP/s/GPU): 26.0 | MFU 8.34% | FLOPs estimator: workload-aware | logical tokens/s: 74564.4 | valid tokens/s: 74564.4 | physical tokens/s: 75036.9 | loss tokens/s: 53006.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.500792E-01 | lm-global-supervised-tokens: 1.192669E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.467960E-02 | lm-token-buffer-mib: 2.333640E+02 | global_load_balancing_loss: 1.000321E+00 | loss scale: 1.0 | grad norm: 0.390 | params norm: 11191.879 | params rms: 0.060600 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:22:34.280183 | finish at 2026-10-17 19:11:37 + [2026-10-13 20:52:47] iteration 875/ 2384 | consumed samples: 28000 | elapsed time per iteration (ms): 224328.7 | throughput per GPU (TFLOP/s/GPU): 25.9 | MFU 8.29% | FLOPs estimator: workload-aware | logical tokens/s: 74788.3 | valid tokens/s: 74788.3 | physical tokens/s: 75150.0 | loss tokens/s: 55258.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.946882E-01 | lm-global-supervised-tokens: 1.239610E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.597067E-02 | lm-token-buffer-mib: 2.322022E+02 | global_load_balancing_loss: 1.001311E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11191.826 | params rms: 0.060600 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:01:52.053730 | finish at 2026-10-17 18:54:39 + [2026-10-13 20:56:28] iteration 876/ 2384 | consumed samples: 28032 | elapsed time per iteration (ms): 221036.2 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.67% | FLOPs estimator: workload-aware | logical tokens/s: 75902.6 | valid tokens/s: 75902.6 | physical tokens/s: 76541.9 | loss tokens/s: 58969.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.567458E-01 | lm-global-supervised-tokens: 1.303448E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.457183E-02 | lm-token-buffer-mib: 2.343786E+02 | global_load_balancing_loss: 1.000939E+00 | loss scale: 1.0 | grad norm: 0.114 | params norm: 11191.780 | params rms: 0.060600 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:35:22.624905 | finish at 2026-10-17 17:31:51 + [2026-10-13 21:00:08] iteration 877/ 2384 | consumed samples: 28064 | elapsed time per iteration (ms): 219482.1 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.75% | FLOPs estimator: workload-aware | logical tokens/s: 76439.9 | valid tokens/s: 76439.9 | physical tokens/s: 77022.4 | loss tokens/s: 58329.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.654578E-01 | lm-global-supervised-tokens: 1.280219E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.353360E-02 | lm-token-buffer-mib: 2.341018E+02 | global_load_balancing_loss: 1.000688E+00 | loss scale: 1.0 | grad norm: 0.129 | params norm: 11191.734 | params rms: 0.060600 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 19:52:39.459682 | finish at 2026-10-17 16:52:47 + [2026-10-13 21:03:49] iteration 878/ 2384 | consumed samples: 28096 | elapsed time per iteration (ms): 220734.7 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.06% | FLOPs estimator: workload-aware | logical tokens/s: 76006.1 | valid tokens/s: 76006.1 | physical tokens/s: 76571.1 | loss tokens/s: 54942.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.954958E-01 | lm-global-supervised-tokens: 1.212768E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.177407E-02 | lm-token-buffer-mib: 2.329806E+02 | global_load_balancing_loss: 1.000210E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11191.693 | params rms: 0.060599 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:20:26.385976 | finish at 2026-10-17 17:24:15 + [2026-10-13 21:07:32] iteration 879/ 2384 | consumed samples: 28128 | elapsed time per iteration (ms): 223724.9 | throughput per GPU (TFLOP/s/GPU): 24.1 | MFU 7.72% | FLOPs estimator: workload-aware | logical tokens/s: 74990.3 | valid tokens/s: 74990.3 | physical tokens/s: 75340.8 | loss tokens/s: 55367.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.831994E-01 | lm-global-supervised-tokens: 1.238699E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.204123E-02 | lm-token-buffer-mib: 2.330636E+02 | global_load_balancing_loss: 1.002308E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11191.658 | params rms: 0.060599 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:31:45.978099 | finish at 2026-10-17 18:39:18 + [2026-10-13 21:11:40] iteration 880/ 2384 | consumed samples: 28160 | elapsed time per iteration (ms): 247426.0 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.41% | FLOPs estimator: workload-aware | logical tokens/s: 67806.9 | valid tokens/s: 67806.9 | physical tokens/s: 68165.8 | loss tokens/s: 48595.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.823836E-01 | lm-global-supervised-tokens: 1.202382E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.746943E-02 | lm-token-buffer-mib: 2.324938E+02 | global_load_balancing_loss: 1.001618E+00 | loss scale: 1.0 | grad norm: 0.139 | params norm: 11191.627 | params rms: 0.060599 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:22:08.695572 | finish at 2026-10-18 04:33:48 +times across ranks (ms): + save-checkpoint: + rank 0: 25768.37 + rank 1: 25768.38 + rank 2: 25768.38 + rank 3: 25768.41 + rank 4: 25768.37 + rank 5: 25768.37 + rank 6: 25768.40 + rank 7: 25768.37 + rank 8: 25768.11 + rank 9: 25768.10 + rank 10: 25768.10 + rank 11: 25768.07 + rank 12: 25768.08 + rank 13: 25768.05 + rank 14: 25768.10 + rank 15: 25768.08 + rank 16: 25768.24 + rank 17: 25768.22 + rank 18: 25768.24 + rank 19: 25768.23 + rank 20: 25768.22 + rank 21: 25768.24 + rank 22: 25768.23 + rank 23: 25768.06 + rank 24: 25768.59 + rank 25: 25768.59 + rank 26: 25768.57 + rank 27: 25768.57 + rank 28: 25768.57 + rank 29: 25768.53 + rank 30: 25768.61 + rank 31: 25768.55 + rank 32: 25768.88 + rank 33: 25768.94 + rank 34: 25768.99 + rank 35: 25768.91 + rank 36: 25768.85 + rank 37: 25768.89 + rank 38: 25768.98 + rank 39: 25768.96 + rank 40: 25767.82 + rank 41: 25767.77 + rank 42: 25767.77 + rank 43: 25767.78 + rank 44: 25767.76 + rank 45: 25767.76 + rank 46: 25767.76 + rank 47: 25767.76 + rank 48: 25767.84 + rank 49: 25767.87 + rank 50: 25767.84 + rank 51: 25767.87 + rank 52: 25767.93 + rank 53: 25767.95 + rank 54: 25767.78 + rank 55: 25767.90 + rank 56: 25768.51 + rank 57: 25768.53 + rank 58: 25768.53 + rank 59: 25768.70 + rank 60: 25768.53 + rank 61: 25768.50 + rank 62: 25768.50 + rank 63: 25768.53 + [2026-10-13 21:15:45] iteration 881/ 2384 | consumed samples: 28192 | elapsed time per iteration (ms): 219061.6 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.77% | FLOPs estimator: workload-aware | logical tokens/s: 76586.6 | valid tokens/s: 76586.6 | physical tokens/s: 77380.5 | loss tokens/s: 54484.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.728944E-01 | lm-global-supervised-tokens: 1.193550E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.963499E-02 | lm-token-buffer-mib: 2.339630E+02 | global_load_balancing_loss: 1.000227E+00 | loss scale: 1.0 | grad norm: 0.160 | params norm: 11191.593 | params rms: 0.060599 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 19:27:29.595147 | finish at 2026-10-17 16:43:14 + failed to log checkpoint /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/iter_0000880 as wandb artifact: Unable to connect to server to verify API token. + [2026-10-13 21:19:37] iteration 882/ 2384 | consumed samples: 28224 | elapsed time per iteration (ms): 232640.8 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.53% | FLOPs estimator: workload-aware | logical tokens/s: 72116.2 | valid tokens/s: 72116.2 | physical tokens/s: 73023.4 | loss tokens/s: 55000.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.867498E-01 | lm-global-supervised-tokens: 1.279532E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.490333E-02 | lm-token-buffer-mib: 2.326576E+02 | global_load_balancing_loss: 9.996476E-01 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11191.565 | params rms: 0.060599 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:03:46.544267 | finish at 2026-10-17 22:23:24 + [2026-10-13 21:23:17] iteration 883/ 2384 | consumed samples: 28256 | elapsed time per iteration (ms): 219815.6 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.85% | FLOPs estimator: workload-aware | logical tokens/s: 76324.0 | valid tokens/s: 76324.0 | physical tokens/s: 77026.6 | loss tokens/s: 54367.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.406625E-01 | lm-global-supervised-tokens: 1.195082E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.341392E-02 | lm-token-buffer-mib: 2.335660E+02 | global_load_balancing_loss: 9.994037E-01 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11191.540 | params rms: 0.060599 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 19:39:03.177544 | finish at 2026-10-17 17:02:20 + [2026-10-13 21:27:00] iteration 884/ 2384 | consumed samples: 28288 | elapsed time per iteration (ms): 222898.2 | throughput per GPU (TFLOP/s/GPU): 26.4 | MFU 8.47% | FLOPs estimator: workload-aware | logical tokens/s: 75268.5 | valid tokens/s: 75268.5 | physical tokens/s: 75922.9 | loss tokens/s: 55467.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.903709E-01 | lm-global-supervised-tokens: 1.236362E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.516680E-02 | lm-token-buffer-mib: 2.357818E+02 | global_load_balancing_loss: 1.000039E+00 | loss scale: 1.0 | grad norm: 0.173 | params norm: 11191.516 | params rms: 0.060598 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:52:27.282171 | finish at 2026-10-17 18:19:27 + [2026-10-13 21:30:41] iteration 885/ 2384 | consumed samples: 28320 | elapsed time per iteration (ms): 221123.7 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.11% | FLOPs estimator: workload-aware | logical tokens/s: 75872.4 | valid tokens/s: 75872.4 | physical tokens/s: 76601.3 | loss tokens/s: 55484.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.253933E-01 | lm-global-supervised-tokens: 1.226901E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.400611E-02 | lm-token-buffer-mib: 2.339400E+02 | global_load_balancing_loss: 9.997057E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11191.489 | params rms: 0.060598 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:04:24.419722 | finish at 2026-10-17 17:35:06 + [2026-10-13 21:34:24] iteration 886/ 2384 | consumed samples: 28352 | elapsed time per iteration (ms): 223018.3 | throughput per GPU (TFLOP/s/GPU): 24.2 | MFU 7.77% | FLOPs estimator: workload-aware | logical tokens/s: 75227.9 | valid tokens/s: 75227.9 | physical tokens/s: 76226.1 | loss tokens/s: 61157.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.875844E-01 | lm-global-supervised-tokens: 1.363932E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.749225E-02 | lm-token-buffer-mib: 2.338306E+02 | global_load_balancing_loss: 1.000563E+00 | loss scale: 1.0 | grad norm: 0.124 | params norm: 11191.460 | params rms: 0.060598 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:48:01.466343 | finish at 2026-10-17 18:22:26 + [2026-10-13 21:38:06] iteration 887/ 2384 | consumed samples: 28384 | elapsed time per iteration (ms): 221477.6 | throughput per GPU (TFLOP/s/GPU): 23.4 | MFU 7.48% | FLOPs estimator: workload-aware | logical tokens/s: 75751.3 | valid tokens/s: 75751.3 | physical tokens/s: 76128.4 | loss tokens/s: 58364.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.633123E-01 | lm-global-supervised-tokens: 1.292645E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.314855E-02 | lm-token-buffer-mib: 2.327530E+02 | global_load_balancing_loss: 1.000245E+00 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11191.433 | params rms: 0.060598 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:05:51.991973 | finish at 2026-10-17 17:43:58 + [2026-10-13 21:41:48] iteration 888/ 2384 | consumed samples: 28416 | elapsed time per iteration (ms): 222607.5 | throughput per GPU (TFLOP/s/GPU): 20.2 | MFU 6.48% | FLOPs estimator: workload-aware | logical tokens/s: 75366.7 | valid tokens/s: 75366.7 | physical tokens/s: 76314.7 | loss tokens/s: 58389.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.242359E-01 | lm-global-supervised-tokens: 1.299796E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.869969E-02 | lm-token-buffer-mib: 2.346669E+02 | global_load_balancing_loss: 1.001881E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11191.405 | params rms: 0.060598 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:30:20.883602 | finish at 2026-10-17 18:12:09 + [2026-10-13 21:45:32] iteration 889/ 2384 | consumed samples: 28448 | elapsed time per iteration (ms): 223998.9 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.43% | FLOPs estimator: workload-aware | logical tokens/s: 74898.6 | valid tokens/s: 74898.6 | physical tokens/s: 75499.2 | loss tokens/s: 59954.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.802638E-01 | lm-global-supervised-tokens: 1.342966E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.331657E-02 | lm-token-buffer-mib: 2.344042E+02 | global_load_balancing_loss: 1.000862E+00 | loss scale: 1.0 | grad norm: 0.126 | params norm: 11191.382 | params rms: 0.060598 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:01:18.335088 | finish at 2026-10-17 18:46:51 + [2026-10-13 21:49:39] iteration 890/ 2384 | consumed samples: 28480 | elapsed time per iteration (ms): 246915.3 | throughput per GPU (TFLOP/s/GPU): 19.7 | MFU 6.30% | FLOPs estimator: workload-aware | logical tokens/s: 67947.2 | valid tokens/s: 67947.2 | physical tokens/s: 68632.1 | loss tokens/s: 46628.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.809579E-01 | lm-global-supervised-tokens: 1.151336E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.678344E-02 | lm-token-buffer-mib: 2.352833E+02 | global_load_balancing_loss: 1.000590E+00 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11191.356 | params rms: 0.060598 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:28:11.406747 | finish at 2026-10-18 04:17:51 + [2026-10-13 21:53:21] iteration 891/ 2384 | consumed samples: 28512 | elapsed time per iteration (ms): 222359.2 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.47% | FLOPs estimator: workload-aware | logical tokens/s: 75450.8 | valid tokens/s: 75450.8 | physical tokens/s: 75902.3 | loss tokens/s: 53091.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.043420E-01 | lm-global-supervised-tokens: 1.180528E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.518803E-02 | lm-token-buffer-mib: 2.337395E+02 | global_load_balancing_loss: 1.000935E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11191.326 | params rms: 0.060597 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:13:02.351810 | finish at 2026-10-17 18:06:24 + [2026-10-13 21:57:09] iteration 892/ 2384 | consumed samples: 28544 | elapsed time per iteration (ms): 227693.1 | throughput per GPU (TFLOP/s/GPU): 25.0 | MFU 8.01% | FLOPs estimator: workload-aware | logical tokens/s: 73683.3 | valid tokens/s: 73683.3 | physical tokens/s: 74296.5 | loss tokens/s: 55706.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.177171E-01 | lm-global-supervised-tokens: 1.268402E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.509786E-02 | lm-token-buffer-mib: 2.344335E+02 | global_load_balancing_loss: 1.001158E+00 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11191.294 | params rms: 0.060597 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:21:58.083465 | finish at 2026-10-17 20:19:07 + [2026-10-13 22:00:54] iteration 893/ 2384 | consumed samples: 28576 | elapsed time per iteration (ms): 224795.5 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.82% | FLOPs estimator: workload-aware | logical tokens/s: 74633.2 | valid tokens/s: 74633.2 | physical tokens/s: 75335.3 | loss tokens/s: 56942.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.506409E-01 | lm-global-supervised-tokens: 1.280050E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.532542E-02 | lm-token-buffer-mib: 2.351598E+02 | global_load_balancing_loss: 9.997296E-01 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11191.260 | params rms: 0.060597 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:06:10.037237 | finish at 2026-10-17 19:07:04 + [2026-10-13 22:04:35] iteration 894/ 2384 | consumed samples: 28608 | elapsed time per iteration (ms): 221076.0 | throughput per GPU (TFLOP/s/GPU): 17.9 | MFU 5.74% | FLOPs estimator: workload-aware | logical tokens/s: 75888.7 | valid tokens/s: 75888.7 | physical tokens/s: 76608.3 | loss tokens/s: 58471.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.922763E-01 | lm-global-supervised-tokens: 1.292655E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.012316E-02 | lm-token-buffer-mib: 2.335753E+02 | global_load_balancing_loss: 9.999449E-01 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11191.228 | params rms: 0.060597 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 19:30:03.271935 | finish at 2026-10-17 17:34:38 + [2026-10-13 22:08:20] iteration 895/ 2384 | consumed samples: 28640 | elapsed time per iteration (ms): 224795.3 | throughput per GPU (TFLOP/s/GPU): 23.9 | MFU 7.67% | FLOPs estimator: workload-aware | logical tokens/s: 74633.2 | valid tokens/s: 74633.2 | physical tokens/s: 75357.0 | loss tokens/s: 58176.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.849827E-01 | lm-global-supervised-tokens: 1.307779E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.365330E-02 | lm-token-buffer-mib: 2.336856E+02 | global_load_balancing_loss: 9.999200E-01 | loss scale: 1.0 | grad norm: 0.233 | params norm: 11191.195 | params rms: 0.060597 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:58:40.187865 | finish at 2026-10-17 19:07:00 + [2026-10-13 22:12:08] iteration 896/ 2384 | consumed samples: 28672 | elapsed time per iteration (ms): 227958.7 | throughput per GPU (TFLOP/s/GPU): 23.9 | MFU 7.67% | FLOPs estimator: workload-aware | logical tokens/s: 73597.5 | valid tokens/s: 73597.5 | physical tokens/s: 75256.0 | loss tokens/s: 55664.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.401963E-01 | lm-global-supervised-tokens: 1.268928E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.517954E-02 | lm-token-buffer-mib: 2.381703E+02 | global_load_balancing_loss: 1.000949E+00 | loss scale: 1.0 | grad norm: 0.181 | params norm: 11191.156 | params rms: 0.060596 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:13:22.527065 | finish at 2026-10-17 20:25:30 + [2026-10-13 22:15:48] iteration 897/ 2384 | consumed samples: 28704 | elapsed time per iteration (ms): 219989.5 | throughput per GPU (TFLOP/s/GPU): 24.1 | MFU 7.71% | FLOPs estimator: workload-aware | logical tokens/s: 76263.6 | valid tokens/s: 76263.6 | physical tokens/s: 76671.6 | loss tokens/s: 54554.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.692822E-01 | lm-global-supervised-tokens: 1.200152E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.326327E-02 | lm-token-buffer-mib: 2.329718E+02 | global_load_balancing_loss: 1.001390E+00 | loss scale: 1.0 | grad norm: 0.125 | params norm: 11191.106 | params rms: 0.060596 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:52:04.452510 | finish at 2026-10-17 17:07:52 + [2026-10-13 22:19:28] iteration 898/ 2384 | consumed samples: 28736 | elapsed time per iteration (ms): 220676.5 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.78% | FLOPs estimator: workload-aware | logical tokens/s: 76026.2 | valid tokens/s: 76026.2 | physical tokens/s: 76577.9 | loss tokens/s: 59344.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.831176E-01 | lm-global-supervised-tokens: 1.309602E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.688940E-02 | lm-token-buffer-mib: 2.333021E+02 | global_load_balancing_loss: 9.998631E-01 | loss scale: 1.0 | grad norm: 0.105 | params norm: 11191.059 | params rms: 0.060596 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 19:05:25.295419 | finish at 2026-10-17 17:24:54 + [2026-10-13 22:23:12] iteration 899/ 2384 | consumed samples: 28768 | elapsed time per iteration (ms): 223798.9 | throughput per GPU (TFLOP/s/GPU): 23.7 | MFU 7.59% | FLOPs estimator: workload-aware | logical tokens/s: 74965.4 | valid tokens/s: 74965.4 | physical tokens/s: 75698.0 | loss tokens/s: 57663.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.693665E-01 | lm-global-supervised-tokens: 1.290506E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.084921E-02 | lm-token-buffer-mib: 2.352783E+02 | global_load_balancing_loss: 1.000371E+00 | loss scale: 1.0 | grad norm: 0.118 | params norm: 11191.017 | params rms: 0.060596 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:19:01.312519 | finish at 2026-10-17 18:42:14 + [2026-10-13 22:27:11] iteration 900/ 2384 | consumed samples: 28800 | elapsed time per iteration (ms): 239047.6 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.73% | FLOPs estimator: workload-aware | logical tokens/s: 70183.5 | valid tokens/s: 70183.5 | physical tokens/s: 70711.5 | loss tokens/s: 54244.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.546417E-01 | lm-global-supervised-tokens: 1.296706E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.385758E-02 | lm-token-buffer-mib: 2.332767E+02 | global_load_balancing_loss: 1.000091E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11190.979 | params rms: 0.060596 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:32:26.712393 | finish at 2026-10-18 00:59:38 + [2026-10-13 22:30:51] iteration 901/ 2384 | consumed samples: 28832 | elapsed time per iteration (ms): 220080.9 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.41% | FLOPs estimator: workload-aware | logical tokens/s: 76231.9 | valid tokens/s: 76231.9 | physical tokens/s: 76760.7 | loss tokens/s: 58729.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.908841E-01 | lm-global-supervised-tokens: 1.292521E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.002482E-02 | lm-token-buffer-mib: 2.336014E+02 | global_load_balancing_loss: 9.990818E-01 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11190.943 | params rms: 0.060595 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:39:39.916645 | finish at 2026-10-17 17:10:31 + [2026-10-13 22:34:34] iteration 902/ 2384 | consumed samples: 28864 | elapsed time per iteration (ms): 222286.2 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.01% | FLOPs estimator: workload-aware | logical tokens/s: 75475.7 | valid tokens/s: 75475.7 | physical tokens/s: 76405.7 | loss tokens/s: 60213.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.827168E-01 | lm-global-supervised-tokens: 1.338462E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.762883E-02 | lm-token-buffer-mib: 2.339707E+02 | global_load_balancing_loss: 1.000978E+00 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11190.908 | params rms: 0.060595 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 19:30:28.154122 | finish at 2026-10-17 18:05:02 + [2026-10-13 22:38:17] iteration 903/ 2384 | consumed samples: 28896 | elapsed time per iteration (ms): 223700.2 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.81% | FLOPs estimator: workload-aware | logical tokens/s: 74998.5 | valid tokens/s: 74998.5 | physical tokens/s: 75459.0 | loss tokens/s: 53985.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.917140E-01 | lm-global-supervised-tokens: 1.207662E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.701684E-02 | lm-token-buffer-mib: 2.335515E+02 | global_load_balancing_loss: 1.000405E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11190.871 | params rms: 0.060595 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:01:40.043988 | finish at 2026-10-17 18:39:57 + [2026-10-13 22:42:04] iteration 904/ 2384 | consumed samples: 28928 | elapsed time per iteration (ms): 226154.3 | throughput per GPU (TFLOP/s/GPU): 24.5 | MFU 7.84% | FLOPs estimator: workload-aware | logical tokens/s: 74184.6 | valid tokens/s: 74184.6 | physical tokens/s: 74998.7 | loss tokens/s: 57352.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.416190E-01 | lm-global-supervised-tokens: 1.297040E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.446326E-02 | lm-token-buffer-mib: 2.353644E+02 | global_load_balancing_loss: 9.998409E-01 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11190.836 | params rms: 0.060595 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:58:28.392544 | finish at 2026-10-17 19:40:32 + [2026-10-13 22:45:50] iteration 905/ 2384 | consumed samples: 28960 | elapsed time per iteration (ms): 226161.9 | throughput per GPU (TFLOP/s/GPU): 20.2 | MFU 6.49% | FLOPs estimator: workload-aware | logical tokens/s: 74182.3 | valid tokens/s: 74182.3 | physical tokens/s: 74956.3 | loss tokens/s: 56632.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.461923E-01 | lm-global-supervised-tokens: 1.280809E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.228195E-02 | lm-token-buffer-mib: 2.343824E+02 | global_load_balancing_loss: 1.000365E+00 | loss scale: 1.0 | grad norm: 0.192 | params norm: 11190.797 | params rms: 0.060595 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:54:53.513283 | finish at 2026-10-17 19:40:43 + [2026-10-13 22:49:35] iteration 906/ 2384 | consumed samples: 28992 | elapsed time per iteration (ms): 225127.2 | throughput per GPU (TFLOP/s/GPU): 24.5 | MFU 7.86% | FLOPs estimator: workload-aware | logical tokens/s: 74523.1 | valid tokens/s: 74523.1 | physical tokens/s: 75199.0 | loss tokens/s: 60275.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.464370E-01 | lm-global-supervised-tokens: 1.356956E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.305378E-02 | lm-token-buffer-mib: 2.346247E+02 | global_load_balancing_loss: 9.993325E-01 | loss scale: 1.0 | grad norm: 0.199 | params norm: 11190.754 | params rms: 0.060594 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:25:38.040197 | finish at 2026-10-17 19:15:13 + [2026-10-13 22:53:20] iteration 907/ 2384 | consumed samples: 29024 | elapsed time per iteration (ms): 225094.6 | throughput per GPU (TFLOP/s/GPU): 26.7 | MFU 8.56% | FLOPs estimator: workload-aware | logical tokens/s: 74534.0 | valid tokens/s: 74534.0 | physical tokens/s: 74791.7 | loss tokens/s: 52553.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.560569E-01 | lm-global-supervised-tokens: 1.182942E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.550317E-02 | lm-token-buffer-mib: 2.329929E+02 | global_load_balancing_loss: 1.000067E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11190.713 | params rms: 0.060594 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:21:04.785769 | finish at 2026-10-17 19:14:25 + [2026-10-13 22:57:07] iteration 908/ 2384 | consumed samples: 29056 | elapsed time per iteration (ms): 227276.2 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.55% | FLOPs estimator: workload-aware | logical tokens/s: 73818.6 | valid tokens/s: 73818.5 | physical tokens/s: 74927.3 | loss tokens/s: 56344.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.102323E-01 | lm-global-supervised-tokens: 1.280571E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.247702E-02 | lm-token-buffer-mib: 2.355745E+02 | global_load_balancing_loss: 1.000489E+00 | loss scale: 1.0 | grad norm: 0.247 | params norm: 11190.669 | params rms: 0.060594 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:10:59.661077 | finish at 2026-10-17 20:08:07 + [2026-10-13 23:00:52] iteration 909/ 2384 | consumed samples: 29088 | elapsed time per iteration (ms): 225072.9 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.95% | FLOPs estimator: workload-aware | logical tokens/s: 74541.1 | valid tokens/s: 74541.1 | physical tokens/s: 75650.5 | loss tokens/s: 56970.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.883194E-01 | lm-global-supervised-tokens: 1.282259E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.343048E-02 | lm-token-buffer-mib: 2.369050E+02 | global_load_balancing_loss: 1.000731E+00 | loss scale: 1.0 | grad norm: 0.608 | params norm: 11190.626 | params rms: 0.060594 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:13:02.456899 | finish at 2026-10-17 19:13:55 + [2026-10-13 23:04:53] iteration 910/ 2384 | consumed samples: 29120 | elapsed time per iteration (ms): 241144.5 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.52% | FLOPs estimator: workload-aware | logical tokens/s: 69573.2 | valid tokens/s: 69573.2 | physical tokens/s: 70149.7 | loss tokens/s: 54946.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.297908E-01 | lm-global-supervised-tokens: 1.325004E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.645217E-02 | lm-token-buffer-mib: 2.337142E+02 | global_load_balancing_loss: 1.001436E+00 | loss scale: 1.0 | grad norm: 0.107 | params norm: 11190.579 | params rms: 0.060593 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:44:07.002162 | finish at 2026-10-18 01:49:00 + [2026-10-13 23:08:41] iteration 911/ 2384 | consumed samples: 29152 | elapsed time per iteration (ms): 227443.3 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.33% | FLOPs estimator: workload-aware | logical tokens/s: 73764.3 | valid tokens/s: 73764.3 | physical tokens/s: 74967.6 | loss tokens/s: 56330.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.587208E-01 | lm-global-supervised-tokens: 1.281209E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.333237E-02 | lm-token-buffer-mib: 2.383914E+02 | global_load_balancing_loss: 1.002516E+00 | loss scale: 1.0 | grad norm: 0.935 | params norm: 11190.534 | params rms: 0.060593 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:03:43.943336 | finish at 2026-10-17 20:12:25 + [2026-10-13 23:12:23] iteration 912/ 2384 | consumed samples: 29184 | elapsed time per iteration (ms): 222524.6 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.26% | FLOPs estimator: workload-aware | logical tokens/s: 75394.7 | valid tokens/s: 75394.7 | physical tokens/s: 75912.0 | loss tokens/s: 55507.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.073929E-01 | lm-global-supervised-tokens: 1.235167E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.253162E-02 | lm-token-buffer-mib: 2.339031E+02 | global_load_balancing_loss: 1.000428E+00 | loss scale: 1.0 | grad norm: 0.166 | params norm: 11190.484 | params rms: 0.060593 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:59:16.283890 | finish at 2026-10-17 18:11:40 + [2026-10-13 23:16:06] iteration 913/ 2384 | consumed samples: 29216 | elapsed time per iteration (ms): 222544.4 | throughput per GPU (TFLOP/s/GPU): 19.3 | MFU 6.18% | FLOPs estimator: workload-aware | logical tokens/s: 75388.1 | valid tokens/s: 75388.1 | physical tokens/s: 75880.8 | loss tokens/s: 55575.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.458323E-01 | lm-global-supervised-tokens: 1.236792E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.991012E-02 | lm-token-buffer-mib: 2.340885E+02 | global_load_balancing_loss: 1.001231E+00 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11190.439 | params rms: 0.060593 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:56:02.867077 | finish at 2026-10-17 18:12:09 + [2026-10-13 23:19:48] iteration 914/ 2384 | consumed samples: 29248 | elapsed time per iteration (ms): 222270.0 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.88% | FLOPs estimator: workload-aware | logical tokens/s: 75481.2 | valid tokens/s: 75481.2 | physical tokens/s: 76088.2 | loss tokens/s: 58993.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.262563E-01 | lm-global-supervised-tokens: 1.311238E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.357074E-02 | lm-token-buffer-mib: 2.339646E+02 | global_load_balancing_loss: 1.000524E+00 | loss scale: 1.0 | grad norm: 0.163 | params norm: 11190.398 | params rms: 0.060592 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:45:36.874037 | finish at 2026-10-17 18:05:25 + [2026-10-13 23:23:31] iteration 915/ 2384 | consumed samples: 29280 | elapsed time per iteration (ms): 222930.3 | throughput per GPU (TFLOP/s/GPU): 25.0 | MFU 8.01% | FLOPs estimator: workload-aware | logical tokens/s: 75257.4 | valid tokens/s: 75257.4 | physical tokens/s: 75762.4 | loss tokens/s: 56793.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.351452E-01 | lm-global-supervised-tokens: 1.266108E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.992407E-02 | lm-token-buffer-mib: 2.330852E+02 | global_load_balancing_loss: 1.000125E+00 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11190.361 | params rms: 0.060592 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:58:04.668835 | finish at 2026-10-17 18:21:36 + [2026-10-13 23:27:12] iteration 916/ 2384 | consumed samples: 29312 | elapsed time per iteration (ms): 221256.2 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.56% | FLOPs estimator: workload-aware | logical tokens/s: 75826.9 | valid tokens/s: 75826.9 | physical tokens/s: 76683.3 | loss tokens/s: 57549.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.360802E-01 | lm-global-supervised-tokens: 1.273307E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.312842E-02 | lm-token-buffer-mib: 2.335447E+02 | global_load_balancing_loss: 9.981647E-01 | loss scale: 1.0 | grad norm: 0.159 | params norm: 11190.321 | params rms: 0.060592 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:13:24.106610 | finish at 2026-10-17 17:40:36 + [2026-10-13 23:30:53] iteration 917/ 2384 | consumed samples: 29344 | elapsed time per iteration (ms): 220940.9 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.29% | FLOPs estimator: workload-aware | logical tokens/s: 75935.3 | valid tokens/s: 75935.3 | physical tokens/s: 76668.2 | loss tokens/s: 53550.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.019585E-01 | lm-global-supervised-tokens: 1.183143E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.305704E-02 | lm-token-buffer-mib: 2.336106E+02 | global_load_balancing_loss: 9.989596E-01 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11190.281 | params rms: 0.060592 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:02:00.284689 | finish at 2026-10-17 17:32:54 + [2026-10-13 23:34:40] iteration 918/ 2384 | consumed samples: 29376 | elapsed time per iteration (ms): 226541.8 | throughput per GPU (TFLOP/s/GPU): 23.2 | MFU 7.45% | FLOPs estimator: workload-aware | logical tokens/s: 74057.8 | valid tokens/s: 74057.8 | physical tokens/s: 74978.6 | loss tokens/s: 55516.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.188198E-01 | lm-global-supervised-tokens: 1.257690E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.642450E-02 | lm-token-buffer-mib: 2.355006E+02 | global_load_balancing_loss: 9.992278E-01 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11190.240 | params rms: 0.060592 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:15:10.265900 | finish at 2026-10-17 19:49:50 + [2026-10-13 23:38:22] iteration 919/ 2384 | consumed samples: 29408 | elapsed time per iteration (ms): 222017.2 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.72% | FLOPs estimator: workload-aware | logical tokens/s: 75567.2 | valid tokens/s: 75567.2 | physical tokens/s: 76431.7 | loss tokens/s: 54620.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.495723E-01 | lm-global-supervised-tokens: 1.212673E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.792304E-02 | lm-token-buffer-mib: 2.335772E+02 | global_load_balancing_loss: 1.000587E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11190.199 | params rms: 0.060591 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:20:55.213708 | finish at 2026-10-17 17:59:17 + [2026-10-13 23:42:24] iteration 920/ 2384 | consumed samples: 29440 | elapsed time per iteration (ms): 241869.1 | throughput per GPU (TFLOP/s/GPU): 19.4 | MFU 6.23% | FLOPs estimator: workload-aware | logical tokens/s: 69364.8 | valid tokens/s: 69364.8 | physical tokens/s: 69831.9 | loss tokens/s: 50063.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.570616E-01 | lm-global-supervised-tokens: 1.210885E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.635202E-02 | lm-token-buffer-mib: 2.330967E+02 | global_load_balancing_loss: 1.002257E+00 | loss scale: 1.0 | grad norm: 0.096 | params norm: 11190.158 | params rms: 0.060591 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:21:36.335661 | finish at 2026-10-18 02:04:00 +times across ranks (ms): + save-checkpoint: + rank 0: 27491.85 + rank 1: 27491.89 + rank 2: 27491.86 + rank 3: 27491.86 + rank 4: 27491.68 + rank 5: 27491.89 + rank 6: 27491.88 + rank 7: 27491.89 + rank 8: 27491.99 + rank 9: 27492.02 + rank 10: 27491.94 + rank 11: 27491.92 + rank 12: 27491.93 + rank 13: 27491.93 + rank 14: 27491.96 + rank 15: 27491.93 + rank 16: 27492.01 + rank 17: 27492.00 + rank 18: 27492.05 + rank 19: 27492.01 + rank 20: 27491.98 + rank 21: 27492.00 + rank 22: 27492.03 + rank 23: 27491.99 + rank 24: 27492.13 + rank 25: 27492.19 + rank 26: 27492.07 + rank 27: 27492.68 + rank 28: 27492.10 + rank 29: 27492.11 + rank 30: 27492.13 + rank 31: 27492.09 + rank 32: 27491.80 + rank 33: 27491.79 + rank 34: 27491.80 + rank 35: 27491.84 + rank 36: 27491.70 + rank 37: 27491.78 + rank 38: 27491.71 + rank 39: 27491.68 + rank 40: 27492.13 + rank 41: 27492.02 + rank 42: 27492.01 + rank 43: 27491.05 + rank 44: 27491.99 + rank 45: 27492.01 + rank 46: 27491.99 + rank 47: 27492.11 + rank 48: 27491.85 + rank 49: 27491.87 + rank 50: 27491.79 + rank 51: 27491.88 + rank 52: 27491.84 + rank 53: 27491.89 + rank 54: 27491.83 + rank 55: 27491.87 + rank 56: 27492.03 + rank 57: 27492.12 + rank 58: 27491.98 + rank 59: 27492.07 + rank 60: 27492.04 + rank 61: 27491.97 + rank 62: 27492.02 + rank 63: 27492.04 + [2026-10-13 23:46:59] iteration 921/ 2384 | consumed samples: 29472 | elapsed time per iteration (ms): 247878.7 | throughput per GPU (TFLOP/s/GPU): 16.3 | MFU 5.23% | FLOPs estimator: workload-aware | logical tokens/s: 67683.1 | valid tokens/s: 67683.1 | physical tokens/s: 68768.6 | loss tokens/s: 53037.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.286181E-01 | lm-global-supervised-tokens: 1.314674E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.964771E+00 | lm-token-buffer-mib: 2.369971E+02 | global_load_balancing_loss: 1.002300E+00 | loss scale: 1.0 | grad norm: 0.202 | params norm: 11190.119 | params rms: 0.060591 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:44:06.591145 | finish at 2026-10-18 04:31:06 + failed to log checkpoint /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/iter_0000920 as wandb artifact: Unable to connect to server to verify API token. + [2026-10-13 23:50:58] iteration 922/ 2384 | consumed samples: 29504 | elapsed time per iteration (ms): 238774.4 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.41% | FLOPs estimator: workload-aware | logical tokens/s: 70263.7 | valid tokens/s: 70263.7 | physical tokens/s: 71017.6 | loss tokens/s: 54039.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.326998E-01 | lm-global-supervised-tokens: 1.290322E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 5.318316E-01 | lm-token-buffer-mib: 2.349522E+02 | global_load_balancing_loss: 1.001323E+00 | loss scale: 1.0 | grad norm: 0.104 | params norm: 11190.079 | params rms: 0.060591 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:58:08.196496 | finish at 2026-10-18 00:49:06 + [2026-10-13 23:54:39] iteration 923/ 2384 | consumed samples: 29536 | elapsed time per iteration (ms): 220727.4 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.88% | FLOPs estimator: workload-aware | logical tokens/s: 76008.7 | valid tokens/s: 76008.7 | physical tokens/s: 76492.7 | loss tokens/s: 57631.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.105130E-01 | lm-global-supervised-tokens: 1.272086E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.081570E-02 | lm-token-buffer-mib: 2.329380E+02 | global_load_balancing_loss: 1.000870E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11190.035 | params rms: 0.060590 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:34:42.674368 | finish at 2026-10-17 17:29:21 + [2026-10-13 23:58:21] iteration 924/ 2384 | consumed samples: 29568 | elapsed time per iteration (ms): 222252.5 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.29% | FLOPs estimator: workload-aware | logical tokens/s: 75487.0 | valid tokens/s: 75487.0 | physical tokens/s: 76407.5 | loss tokens/s: 59054.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.625440E-01 | lm-global-supervised-tokens: 1.312508E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.740819E-02 | lm-token-buffer-mib: 2.361848E+02 | global_load_balancing_loss: 9.999263E-01 | loss scale: 1.0 | grad norm: 0.295 | params norm: 11189.990 | params rms: 0.060590 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:08:08.605180 | finish at 2026-10-17 18:06:30 + [2026-10-14 00:02:03] iteration 925/ 2384 | consumed samples: 29600 | elapsed time per iteration (ms): 221673.7 | throughput per GPU (TFLOP/s/GPU): 19.7 | MFU 6.30% | FLOPs estimator: workload-aware | logical tokens/s: 75684.2 | valid tokens/s: 75684.2 | physical tokens/s: 76353.2 | loss tokens/s: 59230.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.836359E-01 | lm-global-supervised-tokens: 1.312974E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.540161E-02 | lm-token-buffer-mib: 2.334619E+02 | global_load_balancing_loss: 9.992596E-01 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11189.946 | params rms: 0.060590 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:50:21.908262 | finish at 2026-10-17 17:52:24 + [2026-10-14 00:05:50] iteration 926/ 2384 | consumed samples: 29632 | elapsed time per iteration (ms): 227802.9 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.83% | FLOPs estimator: workload-aware | logical tokens/s: 73647.9 | valid tokens/s: 73647.8 | physical tokens/s: 75201.9 | loss tokens/s: 57747.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.311432E-01 | lm-global-supervised-tokens: 1.315514E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.402083E-02 | lm-token-buffer-mib: 2.368562E+02 | global_load_balancing_loss: 9.994730E-01 | loss scale: 1.0 | grad norm: 0.152 | params norm: 11189.900 | params rms: 0.060590 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:15:36.557050 | finish at 2026-10-17 20:21:27 + [2026-10-14 00:09:30] iteration 927/ 2384 | consumed samples: 29664 | elapsed time per iteration (ms): 220094.3 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.45% | FLOPs estimator: workload-aware | logical tokens/s: 76227.3 | valid tokens/s: 76227.2 | physical tokens/s: 76567.3 | loss tokens/s: 55669.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.956798E-01 | lm-global-supervised-tokens: 1.225251E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.283682E-02 | lm-token-buffer-mib: 2.329233E+02 | global_load_balancing_loss: 1.000067E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11189.853 | params rms: 0.060589 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:04:37.361798 | finish at 2026-10-17 17:14:08 + [2026-10-14 00:13:15] iteration 928/ 2384 | consumed samples: 29696 | elapsed time per iteration (ms): 224307.6 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.58% | FLOPs estimator: workload-aware | logical tokens/s: 74795.2 | valid tokens/s: 74795.2 | physical tokens/s: 75823.9 | loss tokens/s: 59087.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.695060E-01 | lm-global-supervised-tokens: 1.325386E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.786767E-02 | lm-token-buffer-mib: 2.341065E+02 | global_load_balancing_loss: 1.000713E+00 | loss scale: 1.0 | grad norm: 0.308 | params norm: 11189.807 | params rms: 0.060589 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:43:11.797592 | finish at 2026-10-17 18:56:27 + [2026-10-14 00:16:59] iteration 929/ 2384 | consumed samples: 29728 | elapsed time per iteration (ms): 224363.0 | throughput per GPU (TFLOP/s/GPU): 23.9 | MFU 7.67% | FLOPs estimator: workload-aware | logical tokens/s: 74776.9 | valid tokens/s: 74776.9 | physical tokens/s: 75218.4 | loss tokens/s: 53915.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.266308E-01 | lm-global-supervised-tokens: 1.209652E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.970670E-02 | lm-token-buffer-mib: 2.338612E+02 | global_load_balancing_loss: 1.001091E+00 | loss scale: 1.0 | grad norm: 0.140 | params norm: 11189.759 | params rms: 0.060589 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:40:48.187079 | finish at 2026-10-17 18:57:47 + [2026-10-14 00:21:01] iteration 930/ 2384 | consumed samples: 29760 | elapsed time per iteration (ms): 241809.1 | throughput per GPU (TFLOP/s/GPU): 18.4 | MFU 5.89% | FLOPs estimator: workload-aware | logical tokens/s: 69382.1 | valid tokens/s: 69382.1 | physical tokens/s: 69944.0 | loss tokens/s: 52520.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.488166E-01 | lm-global-supervised-tokens: 1.269984E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.576408E-02 | lm-token-buffer-mib: 2.335607E+02 | global_load_balancing_loss: 9.994186E-01 | loss scale: 1.0 | grad norm: 0.187 | params norm: 11189.707 | params rms: 0.060589 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:39:50.374074 | finish at 2026-10-18 02:00:51 + [2026-10-14 00:24:47] iteration 931/ 2384 | consumed samples: 29792 | elapsed time per iteration (ms): 225588.1 | throughput per GPU (TFLOP/s/GPU): 23.5 | MFU 7.54% | FLOPs estimator: workload-aware | logical tokens/s: 74370.8 | valid tokens/s: 74370.8 | physical tokens/s: 75022.7 | loss tokens/s: 56220.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.177348E-01 | lm-global-supervised-tokens: 1.268268E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.138128E-02 | lm-token-buffer-mib: 2.334521E+02 | global_load_balancing_loss: 9.998121E-01 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11189.657 | params rms: 0.060588 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 19:02:59.532794 | finish at 2026-10-17 19:27:46 + [2026-10-14 00:28:28] iteration 932/ 2384 | consumed samples: 29824 | elapsed time per iteration (ms): 221454.5 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.15% | FLOPs estimator: workload-aware | logical tokens/s: 75759.2 | valid tokens/s: 75759.2 | physical tokens/s: 76571.0 | loss tokens/s: 58888.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.910605E-01 | lm-global-supervised-tokens: 1.304108E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.443879E-02 | lm-token-buffer-mib: 2.351939E+02 | global_load_balancing_loss: 9.994329E-01 | loss scale: 1.0 | grad norm: 0.150 | params norm: 11189.606 | params rms: 0.060588 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:19:11.919402 | finish at 2026-10-17 17:47:40 + [2026-10-14 00:32:12] iteration 933/ 2384 | consumed samples: 29856 | elapsed time per iteration (ms): 224297.0 | throughput per GPU (TFLOP/s/GPU): 23.9 | MFU 7.66% | FLOPs estimator: workload-aware | logical tokens/s: 74799.0 | valid tokens/s: 74799.0 | physical tokens/s: 75292.5 | loss tokens/s: 57825.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.958532E-01 | lm-global-supervised-tokens: 1.297013E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.517385E-02 | lm-token-buffer-mib: 2.337578E+02 | global_load_balancing_loss: 1.000897E+00 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11189.555 | params rms: 0.060588 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:24:14.940327 | finish at 2026-10-17 18:56:27 + [2026-10-14 00:35:55] iteration 934/ 2384 | consumed samples: 29888 | elapsed time per iteration (ms): 222955.4 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.47% | FLOPs estimator: workload-aware | logical tokens/s: 75249.1 | valid tokens/s: 75249.1 | physical tokens/s: 75952.2 | loss tokens/s: 57252.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.389195E-01 | lm-global-supervised-tokens: 1.276474E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.324584E-02 | lm-token-buffer-mib: 2.327507E+02 | global_load_balancing_loss: 1.000335E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11189.502 | params rms: 0.060588 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:48:05.324454 | finish at 2026-10-17 18:24:01 + [2026-10-14 00:39:39] iteration 935/ 2384 | consumed samples: 29920 | elapsed time per iteration (ms): 223347.7 | throughput per GPU (TFLOP/s/GPU): 20.2 | MFU 6.49% | FLOPs estimator: workload-aware | logical tokens/s: 75117.0 | valid tokens/s: 75117.0 | physical tokens/s: 75642.6 | loss tokens/s: 60718.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.399305E-01 | lm-global-supervised-tokens: 1.356132E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.666839E-02 | lm-token-buffer-mib: 2.341403E+02 | global_load_balancing_loss: 1.001476E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11189.448 | params rms: 0.060587 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:53:50.840964 | finish at 2026-10-17 18:33:29 + [2026-10-14 00:43:21] iteration 936/ 2384 | consumed samples: 29952 | elapsed time per iteration (ms): 221932.4 | throughput per GPU (TFLOP/s/GPU): 24.6 | MFU 7.89% | FLOPs estimator: workload-aware | logical tokens/s: 75596.0 | valid tokens/s: 75596.0 | physical tokens/s: 76446.2 | loss tokens/s: 60030.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.995793E-01 | lm-global-supervised-tokens: 1.332269E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.231640E-02 | lm-token-buffer-mib: 2.331757E+02 | global_load_balancing_loss: 1.001417E+00 | loss scale: 1.0 | grad norm: 0.167 | params norm: 11189.393 | params rms: 0.060587 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:15:58.129683 | finish at 2026-10-17 17:59:19 + [2026-10-14 00:47:02] iteration 937/ 2384 | consumed samples: 29984 | elapsed time per iteration (ms): 221442.5 | throughput per GPU (TFLOP/s/GPU): 18.9 | MFU 6.05% | FLOPs estimator: workload-aware | logical tokens/s: 75763.3 | valid tokens/s: 75763.3 | physical tokens/s: 77164.5 | loss tokens/s: 58325.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.968574E-01 | lm-global-supervised-tokens: 1.291572E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.873230E-02 | lm-token-buffer-mib: 2.348216E+02 | global_load_balancing_loss: 1.001896E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11189.339 | params rms: 0.060587 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:00:27.282831 | finish at 2026-10-17 17:47:29 + [2026-10-14 00:50:48] iteration 938/ 2384 | consumed samples: 30016 | elapsed time per iteration (ms): 226449.0 | throughput per GPU (TFLOP/s/GPU): 24.6 | MFU 7.89% | FLOPs estimator: workload-aware | logical tokens/s: 74088.2 | valid tokens/s: 74088.2 | physical tokens/s: 75073.5 | loss tokens/s: 53812.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.996346E-01 | lm-global-supervised-tokens: 1.218566E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.388886E-02 | lm-token-buffer-mib: 2.369635E+02 | global_load_balancing_loss: 9.990959E-01 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11189.287 | params rms: 0.060586 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:57:25.259000 | finish at 2026-10-17 19:48:14 + [2026-10-14 00:54:32] iteration 939/ 2384 | consumed samples: 30048 | elapsed time per iteration (ms): 223152.9 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.09% | FLOPs estimator: workload-aware | logical tokens/s: 75182.4 | valid tokens/s: 75182.4 | physical tokens/s: 75739.6 | loss tokens/s: 57235.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.468086E-01 | lm-global-supervised-tokens: 1.277219E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.599371E-02 | lm-token-buffer-mib: 2.342024E+02 | global_load_balancing_loss: 9.996639E-01 | loss scale: 1.0 | grad norm: 0.122 | params norm: 11189.235 | params rms: 0.060586 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:34:16.004552 | finish at 2026-10-17 18:28:48 + [2026-10-14 00:58:34] iteration 940/ 2384 | consumed samples: 30080 | elapsed time per iteration (ms): 242860.1 | throughput per GPU (TFLOP/s/GPU): 24.5 | MFU 7.84% | FLOPs estimator: workload-aware | logical tokens/s: 69081.5 | valid tokens/s: 69081.5 | physical tokens/s: 69617.3 | loss tokens/s: 48705.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.849753E-01 | lm-global-supervised-tokens: 1.182863E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.635447E-02 | lm-token-buffer-mib: 2.331295E+02 | global_load_balancing_loss: 1.000539E+00 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11189.179 | params rms: 0.060586 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:24:50.031610 | finish at 2026-10-18 02:23:24 + [2026-10-14 01:02:19] iteration 941/ 2384 | consumed samples: 30112 | elapsed time per iteration (ms): 224985.0 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.30% | FLOPs estimator: workload-aware | logical tokens/s: 74570.3 | valid tokens/s: 74570.3 | physical tokens/s: 75320.8 | loss tokens/s: 55991.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.969975E-01 | lm-global-supervised-tokens: 1.259728E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.993397E-02 | lm-token-buffer-mib: 2.344351E+02 | global_load_balancing_loss: 1.001723E+00 | loss scale: 1.0 | grad norm: 0.140 | params norm: 11189.121 | params rms: 0.060585 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:10:53.407142 | finish at 2026-10-17 19:13:13 + [2026-10-14 01:06:01] iteration 942/ 2384 | consumed samples: 30144 | elapsed time per iteration (ms): 221677.3 | throughput per GPU (TFLOP/s/GPU): 26.9 | MFU 8.63% | FLOPs estimator: workload-aware | logical tokens/s: 75682.9 | valid tokens/s: 75682.9 | physical tokens/s: 75922.4 | loss tokens/s: 58589.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.581162E-01 | lm-global-supervised-tokens: 1.298800E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.254142E-02 | lm-token-buffer-mib: 2.324583E+02 | global_load_balancing_loss: 9.992710E-01 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11189.066 | params rms: 0.060585 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:47:38.679974 | finish at 2026-10-17 17:53:40 + [2026-10-14 01:09:43] iteration 943/ 2384 | consumed samples: 30176 | elapsed time per iteration (ms): 221695.3 | throughput per GPU (TFLOP/s/GPU): 23.9 | MFU 7.66% | FLOPs estimator: workload-aware | logical tokens/s: 75676.8 | valid tokens/s: 75676.8 | physical tokens/s: 76064.0 | loss tokens/s: 56555.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.526870E-01 | lm-global-supervised-tokens: 1.253817E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.500707E-02 | lm-token-buffer-mib: 2.329872E+02 | global_load_balancing_loss: 1.001007E+00 | loss scale: 1.0 | grad norm: 0.159 | params norm: 11189.014 | params rms: 0.060585 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:44:22.929509 | finish at 2026-10-17 17:54:06 + [2026-10-14 01:13:24] iteration 944/ 2384 | consumed samples: 30208 | elapsed time per iteration (ms): 221390.2 | throughput per GPU (TFLOP/s/GPU): 23.8 | MFU 7.62% | FLOPs estimator: workload-aware | logical tokens/s: 75780.9 | valid tokens/s: 75780.9 | physical tokens/s: 76406.8 | loss tokens/s: 57027.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.981419E-01 | lm-global-supervised-tokens: 1.262535E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.306503E-02 | lm-token-buffer-mib: 2.340311E+02 | global_load_balancing_loss: 1.001067E+00 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11188.962 | params rms: 0.060585 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:33:21.934204 | finish at 2026-10-17 17:46:46 + [2026-10-14 01:17:08] iteration 945/ 2384 | consumed samples: 30240 | elapsed time per iteration (ms): 223493.5 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.72% | FLOPs estimator: workload-aware | logical tokens/s: 75067.8 | valid tokens/s: 75067.8 | physical tokens/s: 75981.2 | loss tokens/s: 62196.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.872311E-01 | lm-global-supervised-tokens: 1.390053E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.401260E-02 | lm-token-buffer-mib: 2.347220E+02 | global_load_balancing_loss: 9.996409E-01 | loss scale: 1.0 | grad norm: 0.119 | params norm: 11188.911 | params rms: 0.060584 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:20:07.199327 | finish at 2026-10-17 18:37:15 + [2026-10-14 01:20:54] iteration 946/ 2384 | consumed samples: 30272 | elapsed time per iteration (ms): 225866.9 | throughput per GPU (TFLOP/s/GPU): 25.0 | MFU 8.00% | FLOPs estimator: workload-aware | logical tokens/s: 74279.0 | valid tokens/s: 74279.0 | physical tokens/s: 75176.0 | loss tokens/s: 54517.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.781358E-01 | lm-global-supervised-tokens: 1.231376E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.429176E-02 | lm-token-buffer-mib: 2.338056E+02 | global_load_balancing_loss: 9.999220E-01 | loss scale: 1.0 | grad norm: 0.137 | params norm: 11188.862 | params rms: 0.060584 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:13:16.672779 | finish at 2026-10-17 19:34:10 + [2026-10-14 01:24:38] iteration 947/ 2384 | consumed samples: 30304 | elapsed time per iteration (ms): 224899.7 | throughput per GPU (TFLOP/s/GPU): 19.0 | MFU 6.09% | FLOPs estimator: workload-aware | logical tokens/s: 74598.5 | valid tokens/s: 74598.5 | physical tokens/s: 75779.6 | loss tokens/s: 57541.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.693775E-01 | lm-global-supervised-tokens: 1.294111E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.774474E-02 | lm-token-buffer-mib: 2.360965E+02 | global_load_balancing_loss: 1.000148E+00 | loss scale: 1.0 | grad norm: 0.126 | params norm: 11188.811 | params rms: 0.060584 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:46:20.833506 | finish at 2026-10-17 19:10:59 + [2026-10-14 01:28:20] iteration 948/ 2384 | consumed samples: 30336 | elapsed time per iteration (ms): 221217.3 | throughput per GPU (TFLOP/s/GPU): 24.0 | MFU 7.69% | FLOPs estimator: workload-aware | logical tokens/s: 75840.3 | valid tokens/s: 75840.3 | physical tokens/s: 76675.4 | loss tokens/s: 57230.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.211838E-01 | lm-global-supervised-tokens: 1.266029E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.514222E-02 | lm-token-buffer-mib: 2.348951E+02 | global_load_balancing_loss: 1.001047E+00 | loss scale: 1.0 | grad norm: 0.169 | params norm: 11188.758 | params rms: 0.060583 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:14:27.977319 | finish at 2026-10-17 17:42:48 + [2026-10-14 01:32:02] iteration 949/ 2384 | consumed samples: 30368 | elapsed time per iteration (ms): 222656.3 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.99% | FLOPs estimator: workload-aware | logical tokens/s: 75350.1 | valid tokens/s: 75350.1 | physical tokens/s: 76044.2 | loss tokens/s: 56590.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.862845E-01 | lm-global-supervised-tokens: 1.260027E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.627495E-02 | lm-token-buffer-mib: 2.334266E+02 | global_load_balancing_loss: 1.001273E+00 | loss scale: 1.0 | grad norm: 0.211 | params norm: 11188.706 | params rms: 0.060583 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:45:11.789913 | finish at 2026-10-17 18:17:14 + [2026-10-14 01:36:07] iteration 950/ 2384 | consumed samples: 30400 | elapsed time per iteration (ms): 244907.8 | throughput per GPU (TFLOP/s/GPU): 23.5 | MFU 7.52% | FLOPs estimator: workload-aware | logical tokens/s: 68504.1 | valid tokens/s: 68504.1 | physical tokens/s: 69014.1 | loss tokens/s: 46671.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.392537E-01 | lm-global-supervised-tokens: 1.143026E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.127603E-02 | lm-token-buffer-mib: 2.339703E+02 | global_load_balancing_loss: 1.001337E+00 | loss scale: 1.0 | grad norm: 0.207 | params norm: 11188.652 | params rms: 0.060583 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:33:17.763260 | finish at 2026-10-18 03:09:25 + [2026-10-14 01:39:50] iteration 951/ 2384 | consumed samples: 30432 | elapsed time per iteration (ms): 222301.9 | throughput per GPU (TFLOP/s/GPU): 18.3 | MFU 5.87% | FLOPs estimator: workload-aware | logical tokens/s: 75470.3 | valid tokens/s: 75470.2 | physical tokens/s: 76410.7 | loss tokens/s: 58545.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.856326E-01 | lm-global-supervised-tokens: 1.301473E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.910176E-02 | lm-token-buffer-mib: 2.357096E+02 | global_load_balancing_loss: 1.000431E+00 | loss scale: 1.0 | grad norm: 0.243 | params norm: 11188.598 | params rms: 0.060583 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:29:18.599339 | finish at 2026-10-17 18:09:08 + [2026-10-14 01:43:29] iteration 952/ 2384 | consumed samples: 30464 | elapsed time per iteration (ms): 219642.4 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.20% | FLOPs estimator: workload-aware | logical tokens/s: 76384.0 | valid tokens/s: 76384.0 | physical tokens/s: 77365.9 | loss tokens/s: 61896.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.071424E-01 | lm-global-supervised-tokens: 1.359511E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.912709E-02 | lm-token-buffer-mib: 2.335370E+02 | global_load_balancing_loss: 1.000179E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11188.548 | params rms: 0.060582 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:22:07.850603 | finish at 2026-10-17 17:05:37 + [2026-10-14 01:47:11] iteration 953/ 2384 | consumed samples: 30496 | elapsed time per iteration (ms): 221362.9 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.07% | FLOPs estimator: workload-aware | logical tokens/s: 75790.4 | valid tokens/s: 75790.4 | physical tokens/s: 76339.3 | loss tokens/s: 54209.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.835454E-01 | lm-global-supervised-tokens: 1.199995E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.603588E-02 | lm-token-buffer-mib: 2.343965E+02 | global_load_balancing_loss: 9.998122E-01 | loss scale: 1.0 | grad norm: 0.188 | params norm: 11188.496 | params rms: 0.060582 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:59:30.332786 | finish at 2026-10-17 17:46:41 + [2026-10-14 01:50:55] iteration 954/ 2384 | consumed samples: 30528 | elapsed time per iteration (ms): 224531.1 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.24% | FLOPs estimator: workload-aware | logical tokens/s: 74721.0 | valid tokens/s: 74721.0 | physical tokens/s: 75565.1 | loss tokens/s: 57533.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.748905E-01 | lm-global-supervised-tokens: 1.291816E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.768963E-02 | lm-token-buffer-mib: 2.353913E+02 | global_load_balancing_loss: 9.989998E-01 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11188.448 | params rms: 0.060582 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:11:19.527941 | finish at 2026-10-17 19:02:15 + [2026-10-14 01:54:38] iteration 955/ 2384 | consumed samples: 30560 | elapsed time per iteration (ms): 222777.9 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.56% | FLOPs estimator: workload-aware | logical tokens/s: 75309.2 | valid tokens/s: 75309.2 | physical tokens/s: 76061.8 | loss tokens/s: 58083.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.482181E-01 | lm-global-supervised-tokens: 1.293973E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.284581E-02 | lm-token-buffer-mib: 2.343195E+02 | global_load_balancing_loss: 9.993736E-01 | loss scale: 1.0 | grad norm: 0.385 | params norm: 11188.400 | params rms: 0.060582 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:25:49.608170 | finish at 2026-10-17 18:20:27 + [2026-10-14 01:58:23] iteration 956/ 2384 | consumed samples: 30592 | elapsed time per iteration (ms): 224699.0 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.99% | FLOPs estimator: workload-aware | logical tokens/s: 74665.2 | valid tokens/s: 74665.2 | physical tokens/s: 75448.8 | loss tokens/s: 53000.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.779208E-01 | lm-global-supervised-tokens: 1.190915E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.524312E-02 | lm-token-buffer-mib: 2.350768E+02 | global_load_balancing_loss: 1.000494E+00 | loss scale: 1.0 | grad norm: 0.229 | params norm: 11188.348 | params rms: 0.060581 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:07:50.216772 | finish at 2026-10-17 19:06:13 + [2026-10-14 02:02:02] iteration 957/ 2384 | consumed samples: 30624 | elapsed time per iteration (ms): 219610.7 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.63% | FLOPs estimator: workload-aware | logical tokens/s: 76395.2 | valid tokens/s: 76395.2 | physical tokens/s: 76789.3 | loss tokens/s: 59052.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.670730E-01 | lm-global-supervised-tokens: 1.296857E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.269624E-02 | lm-token-buffer-mib: 2.330789E+02 | global_load_balancing_loss: 1.001611E+00 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11188.296 | params rms: 0.060581 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:03:04.461601 | finish at 2026-10-17 17:05:07 + [2026-10-14 02:05:44] iteration 958/ 2384 | consumed samples: 30656 | elapsed time per iteration (ms): 221406.2 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.08% | FLOPs estimator: workload-aware | logical tokens/s: 75775.7 | valid tokens/s: 75775.7 | physical tokens/s: 76585.4 | loss tokens/s: 58626.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.658139E-01 | lm-global-supervised-tokens: 1.298032E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.295907E-02 | lm-token-buffer-mib: 2.347370E+02 | global_load_balancing_loss: 1.002150E+00 | loss scale: 1.0 | grad norm: 0.115 | params norm: 11188.247 | params rms: 0.060581 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:42:05.278162 | finish at 2026-10-17 17:47:49 + [2026-10-14 02:09:27] iteration 959/ 2384 | consumed samples: 30688 | elapsed time per iteration (ms): 223849.6 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 7.00% | FLOPs estimator: workload-aware | logical tokens/s: 74948.5 | valid tokens/s: 74948.5 | physical tokens/s: 75872.1 | loss tokens/s: 53174.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.344902E-01 | lm-global-supervised-tokens: 1.190301E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.552953E-02 | lm-token-buffer-mib: 2.343716E+02 | global_load_balancing_loss: 1.001353E+00 | loss scale: 1.0 | grad norm: 0.283 | params norm: 11188.201 | params rms: 0.060580 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:36:25.615218 | finish at 2026-10-17 18:45:53 + [2026-10-14 02:13:33] iteration 960/ 2384 | consumed samples: 30720 | elapsed time per iteration (ms): 245795.5 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.69% | FLOPs estimator: workload-aware | logical tokens/s: 68256.7 | valid tokens/s: 68256.7 | physical tokens/s: 68898.9 | loss tokens/s: 54141.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.447038E-01 | lm-global-supervised-tokens: 1.330778E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.159541E-02 | lm-token-buffer-mib: 2.337471E+02 | global_load_balancing_loss: 1.000681E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11188.155 | params rms: 0.060580 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:13:32.783569 | finish at 2026-10-18 03:27:06 +times across ranks (ms): + save-checkpoint: + rank 0: 24954.80 + rank 1: 24954.71 + rank 2: 24954.76 + rank 3: 24954.75 + rank 4: 24954.73 + rank 5: 24954.75 + rank 6: 24954.79 + rank 7: 24954.72 + rank 8: 24953.88 + rank 9: 24953.93 + rank 10: 24953.94 + rank 11: 24953.90 + rank 12: 24953.89 + rank 13: 24953.87 + rank 14: 24953.92 + rank 15: 24953.88 + rank 16: 24954.43 + rank 17: 24954.38 + rank 18: 24954.38 + rank 19: 24954.37 + rank 20: 24954.36 + rank 21: 24954.35 + rank 22: 24954.40 + rank 23: 24954.32 + rank 24: 24954.97 + rank 25: 24954.95 + rank 26: 24954.93 + rank 27: 24955.13 + rank 28: 24954.94 + rank 29: 24954.95 + rank 30: 24954.98 + rank 31: 24954.93 + rank 32: 24954.87 + rank 33: 24954.85 + rank 34: 24954.87 + rank 35: 24954.92 + rank 36: 24954.82 + rank 37: 24954.92 + rank 38: 24954.88 + rank 39: 24954.77 + rank 40: 24954.56 + rank 41: 24954.71 + rank 42: 24954.71 + rank 43: 24954.69 + rank 44: 24954.71 + rank 45: 24954.71 + rank 46: 24954.71 + rank 47: 24954.71 + rank 48: 24954.66 + rank 49: 24954.82 + rank 50: 24954.71 + rank 51: 24954.91 + rank 52: 24954.69 + rank 53: 24954.83 + rank 54: 24954.68 + rank 55: 24954.65 + rank 56: 24954.97 + rank 57: 24954.96 + rank 58: 24954.98 + rank 59: 24954.97 + rank 60: 24954.97 + rank 61: 24954.97 + rank 62: 24954.96 + rank 63: 24954.96 + [2026-10-14 02:17:55] iteration 961/ 2384 | consumed samples: 30752 | elapsed time per iteration (ms): 236348.5 | throughput per GPU (TFLOP/s/GPU): 19.0 | MFU 6.09% | FLOPs estimator: workload-aware | logical tokens/s: 70985.0 | valid tokens/s: 70985.0 | physical tokens/s: 72303.5 | loss tokens/s: 54584.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.465574E-01 | lm-global-supervised-tokens: 1.290090E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 6.604902E-01 | lm-token-buffer-mib: 2.366473E+02 | global_load_balancing_loss: 1.000365E+00 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11188.109 | params rms: 0.060580 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:25:23.975570 | finish at 2026-10-17 23:43:19 + [2026-10-14 02:21:45] iteration 962/ 2384 | consumed samples: 30784 | elapsed time per iteration (ms): 230765.8 | throughput per GPU (TFLOP/s/GPU): 17.5 | MFU 5.61% | FLOPs estimator: workload-aware | logical tokens/s: 72702.4 | valid tokens/s: 72702.4 | physical tokens/s: 73829.5 | loss tokens/s: 55028.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.194007E-01 | lm-global-supervised-tokens: 1.269873E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.747158E-02 | lm-token-buffer-mib: 2.366097E+02 | global_load_balancing_loss: 9.999337E-01 | loss scale: 1.0 | grad norm: 0.225 | params norm: 11188.063 | params rms: 0.060580 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 19:09:08.925898 | finish at 2026-10-17 21:30:54 + [2026-10-14 02:25:26] iteration 963/ 2384 | consumed samples: 30816 | elapsed time per iteration (ms): 221093.8 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.61% | FLOPs estimator: workload-aware | logical tokens/s: 75882.7 | valid tokens/s: 75882.7 | physical tokens/s: 76681.4 | loss tokens/s: 59493.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.671415E-01 | lm-global-supervised-tokens: 1.315365E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.281024E-02 | lm-token-buffer-mib: 2.350346E+02 | global_load_balancing_loss: 9.980396E-01 | loss scale: 1.0 | grad norm: 0.175 | params norm: 11188.018 | params rms: 0.060579 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:16:14.304465 | finish at 2026-10-17 17:41:41 + [2026-10-14 02:29:07] iteration 964/ 2384 | consumed samples: 30848 | elapsed time per iteration (ms): 220516.5 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.06% | FLOPs estimator: workload-aware | logical tokens/s: 76081.3 | valid tokens/s: 76081.3 | physical tokens/s: 76693.0 | loss tokens/s: 56531.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.576753E-01 | lm-global-supervised-tokens: 1.246612E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.712527E-02 | lm-token-buffer-mib: 2.337939E+02 | global_load_balancing_loss: 9.990905E-01 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11187.973 | params rms: 0.060579 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:58:53.471298 | finish at 2026-10-17 17:28:00 + [2026-10-14 02:32:48] iteration 965/ 2384 | consumed samples: 30880 | elapsed time per iteration (ms): 221143.1 | throughput per GPU (TFLOP/s/GPU): 23.9 | MFU 7.65% | FLOPs estimator: workload-aware | logical tokens/s: 75865.8 | valid tokens/s: 75865.8 | physical tokens/s: 76587.9 | loss tokens/s: 57597.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.848649E-01 | lm-global-supervised-tokens: 1.273720E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.282414E-02 | lm-token-buffer-mib: 2.342774E+02 | global_load_balancing_loss: 9.995283E-01 | loss scale: 1.0 | grad norm: 0.122 | params norm: 11187.927 | params rms: 0.060579 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:10:02.073142 | finish at 2026-10-17 17:42:50 + [2026-10-14 02:36:28] iteration 966/ 2384 | consumed samples: 30912 | elapsed time per iteration (ms): 219906.8 | throughput per GPU (TFLOP/s/GPU): 23.5 | MFU 7.54% | FLOPs estimator: workload-aware | logical tokens/s: 76292.2 | valid tokens/s: 76292.2 | physical tokens/s: 76891.9 | loss tokens/s: 57281.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.476559E-01 | lm-global-supervised-tokens: 1.259662E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.453116E-02 | lm-token-buffer-mib: 2.345425E+02 | global_load_balancing_loss: 1.001192E+00 | loss scale: 1.0 | grad norm: 0.123 | params norm: 11187.878 | params rms: 0.060579 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:37:07.831626 | finish at 2026-10-17 17:13:36 + [2026-10-14 02:40:13] iteration 967/ 2384 | consumed samples: 30944 | elapsed time per iteration (ms): 225361.5 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.83% | FLOPs estimator: workload-aware | logical tokens/s: 74445.8 | valid tokens/s: 74445.8 | physical tokens/s: 75386.7 | loss tokens/s: 55477.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.564134E-01 | lm-global-supervised-tokens: 1.250259E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.334731E-02 | lm-token-buffer-mib: 2.373901E+02 | global_load_balancing_loss: 1.001539E+00 | loss scale: 1.0 | grad norm: 0.289 | params norm: 11187.833 | params rms: 0.060578 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:42:17.199927 | finish at 2026-10-17 19:22:31 + [2026-10-14 02:43:57] iteration 968/ 2384 | consumed samples: 30976 | elapsed time per iteration (ms): 223427.3 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.91% | FLOPs estimator: workload-aware | logical tokens/s: 75090.2 | valid tokens/s: 75090.2 | physical tokens/s: 75362.4 | loss tokens/s: 54501.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.977483E-01 | lm-global-supervised-tokens: 1.217707E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.323371E-02 | lm-token-buffer-mib: 2.328437E+02 | global_load_balancing_loss: 1.002529E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11187.792 | params rms: 0.060578 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:52:53.060143 | finish at 2026-10-17 18:36:50 + [2026-10-14 02:47:38] iteration 969/ 2384 | consumed samples: 31008 | elapsed time per iteration (ms): 221478.3 | throughput per GPU (TFLOP/s/GPU): 27.3 | MFU 8.74% | FLOPs estimator: workload-aware | logical tokens/s: 75750.9 | valid tokens/s: 75750.9 | physical tokens/s: 76243.4 | loss tokens/s: 56267.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.243350E-01 | lm-global-supervised-tokens: 1.246201E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.579664E-02 | lm-token-buffer-mib: 2.337478E+02 | global_load_balancing_loss: 1.001730E+00 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11187.754 | params rms: 0.060578 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:03:11.864805 | finish at 2026-10-17 17:50:50 + [2026-10-14 02:51:42] iteration 970/ 2384 | consumed samples: 31040 | elapsed time per iteration (ms): 243300.7 | throughput per GPU (TFLOP/s/GPU): 19.7 | MFU 6.32% | FLOPs estimator: workload-aware | logical tokens/s: 68956.6 | valid tokens/s: 68956.6 | physical tokens/s: 69712.2 | loss tokens/s: 52529.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.953471E-01 | lm-global-supervised-tokens: 1.278049E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.201477E-02 | lm-token-buffer-mib: 2.358062E+02 | global_load_balancing_loss: 9.992823E-01 | loss scale: 1.0 | grad norm: 0.282 | params norm: 11187.717 | params rms: 0.060578 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 23:33:47.187706 | finish at 2026-10-18 02:25:29 + [2026-10-14 02:55:22] iteration 971/ 2384 | consumed samples: 31072 | elapsed time per iteration (ms): 220109.7 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.39% | FLOPs estimator: workload-aware | logical tokens/s: 76222.0 | valid tokens/s: 76222.0 | physical tokens/s: 76604.1 | loss tokens/s: 54372.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.560659E-01 | lm-global-supervised-tokens: 1.196780E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.561034E-02 | lm-token-buffer-mib: 2.325854E+02 | global_load_balancing_loss: 9.989921E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11187.677 | params rms: 0.060578 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:23:34.975056 | finish at 2026-10-17 17:18:57 + [2026-10-14 02:59:46] iteration 972/ 2384 | consumed samples: 31104 | elapsed time per iteration (ms): 264761.7 | throughput per GPU (TFLOP/s/GPU): 17.8 | MFU 5.71% | FLOPs estimator: workload-aware | logical tokens/s: 63367.2 | valid tokens/s: 63367.2 | physical tokens/s: 64946.2 | loss tokens/s: 46106.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.819838E-01 | lm-global-supervised-tokens: 1.220714E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.872401E-02 | lm-token-buffer-mib: 2.348354E+02 | global_load_balancing_loss: 1.000346E+00 | loss scale: 1.0 | grad norm: 0.302 | params norm: 11187.635 | params rms: 0.060577 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:50:43.464733 | finish at 2026-10-18 10:50:30 + [2026-10-14 03:03:29] iteration 973/ 2384 | consumed samples: 31136 | elapsed time per iteration (ms): 222860.3 | throughput per GPU (TFLOP/s/GPU): 19.7 | MFU 6.32% | FLOPs estimator: workload-aware | logical tokens/s: 75281.2 | valid tokens/s: 75281.2 | physical tokens/s: 76799.9 | loss tokens/s: 58032.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.607485E-01 | lm-global-supervised-tokens: 1.293303E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.034507E-02 | lm-token-buffer-mib: 2.346877E+02 | global_load_balancing_loss: 9.988881E-01 | loss scale: 1.0 | grad norm: 0.267 | params norm: 11187.592 | params rms: 0.060577 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:20:55.903239 | finish at 2026-10-17 18:24:25 + [2026-10-14 03:07:11] iteration 974/ 2384 | consumed samples: 31168 | elapsed time per iteration (ms): 221820.8 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.59% | FLOPs estimator: workload-aware | logical tokens/s: 75634.1 | valid tokens/s: 75634.1 | physical tokens/s: 76095.2 | loss tokens/s: 58942.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.954030E-01 | lm-global-supervised-tokens: 1.307464E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.610447E-02 | lm-token-buffer-mib: 2.338861E+02 | global_load_balancing_loss: 9.996139E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11187.553 | params rms: 0.060577 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:52:47.259178 | finish at 2026-10-17 17:59:58 + [2026-10-14 03:10:53] iteration 975/ 2384 | consumed samples: 31200 | elapsed time per iteration (ms): 222026.8 | throughput per GPU (TFLOP/s/GPU): 23.4 | MFU 7.49% | FLOPs estimator: workload-aware | logical tokens/s: 75563.7 | valid tokens/s: 75563.7 | physical tokens/s: 76345.7 | loss tokens/s: 60898.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.505821E-01 | lm-global-supervised-tokens: 1.352102E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.401043E-02 | lm-token-buffer-mib: 2.337649E+02 | global_load_balancing_loss: 9.990566E-01 | loss scale: 1.0 | grad norm: 0.166 | params norm: 11187.515 | params rms: 0.060577 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:53:55.738240 | finish at 2026-10-17 18:04:49 + [2026-10-14 03:14:39] iteration 976/ 2384 | consumed samples: 31232 | elapsed time per iteration (ms): 225577.4 | throughput per GPU (TFLOP/s/GPU): 25.1 | MFU 8.03% | FLOPs estimator: workload-aware | logical tokens/s: 74374.2 | valid tokens/s: 74374.2 | physical tokens/s: 75103.7 | loss tokens/s: 56011.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.220142E-01 | lm-global-supervised-tokens: 1.263490E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.438884E-02 | lm-token-buffer-mib: 2.347813E+02 | global_load_balancing_loss: 1.000093E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11187.476 | params rms: 0.060577 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:13:32.919739 | finish at 2026-10-17 19:28:12 + [2026-10-14 03:18:20] iteration 977/ 2384 | consumed samples: 31264 | elapsed time per iteration (ms): 221292.5 | throughput per GPU (TFLOP/s/GPU): 19.0 | MFU 6.08% | FLOPs estimator: workload-aware | logical tokens/s: 75814.6 | valid tokens/s: 75814.6 | physical tokens/s: 76850.3 | loss tokens/s: 59986.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.467901E-01 | lm-global-supervised-tokens: 1.327448E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.538828E-02 | lm-token-buffer-mib: 2.338451E+02 | global_load_balancing_loss: 1.003268E+00 | loss scale: 1.0 | grad norm: 0.176 | params norm: 11187.437 | params rms: 0.060576 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:29:18.590801 | finish at 2026-10-17 17:47:39 + [2026-10-14 03:22:02] iteration 978/ 2384 | consumed samples: 31296 | elapsed time per iteration (ms): 221916.6 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.98% | FLOPs estimator: workload-aware | logical tokens/s: 75601.4 | valid tokens/s: 75601.4 | physical tokens/s: 76169.9 | loss tokens/s: 56441.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.588662E-01 | lm-global-supervised-tokens: 1.252522E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.518699E-02 | lm-token-buffer-mib: 2.330396E+02 | global_load_balancing_loss: 1.002151E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11187.398 | params rms: 0.060576 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:40:14.672541 | finish at 2026-10-17 18:02:17 + [2026-10-14 03:25:42] iteration 979/ 2384 | consumed samples: 31328 | elapsed time per iteration (ms): 220383.1 | throughput per GPU (TFLOP/s/GPU): 24.7 | MFU 7.93% | FLOPs estimator: workload-aware | logical tokens/s: 76127.4 | valid tokens/s: 76127.4 | physical tokens/s: 76677.0 | loss tokens/s: 54367.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.292534E-01 | lm-global-supervised-tokens: 1.198158E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.912393E-02 | lm-token-buffer-mib: 2.338107E+02 | global_load_balancing_loss: 1.000614E+00 | loss scale: 1.0 | grad norm: 0.118 | params norm: 11187.359 | params rms: 0.060576 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:00:38.197595 | finish at 2026-10-17 17:26:20 + [2026-10-14 03:29:44] iteration 980/ 2384 | consumed samples: 31360 | elapsed time per iteration (ms): 242204.4 | throughput per GPU (TFLOP/s/GPU): 18.3 | MFU 5.87% | FLOPs estimator: workload-aware | logical tokens/s: 69268.8 | valid tokens/s: 69268.8 | physical tokens/s: 69790.7 | loss tokens/s: 50166.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.939594E-01 | lm-global-supervised-tokens: 1.215065E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.671375E-02 | lm-token-buffer-mib: 2.334621E+02 | global_load_balancing_loss: 1.000986E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11187.316 | params rms: 0.060576 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:27:34.918774 | finish at 2026-10-18 01:57:19 + [2026-10-14 03:33:26] iteration 981/ 2384 | consumed samples: 31392 | elapsed time per iteration (ms): 221964.6 | throughput per GPU (TFLOP/s/GPU): 18.9 | MFU 6.07% | FLOPs estimator: workload-aware | logical tokens/s: 75585.0 | valid tokens/s: 75585.0 | physical tokens/s: 76361.6 | loss tokens/s: 57720.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.714535E-01 | lm-global-supervised-tokens: 1.281199E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.454137E-02 | lm-token-buffer-mib: 2.346338E+02 | global_load_balancing_loss: 1.000437E+00 | loss scale: 1.0 | grad norm: 0.164 | params norm: 11187.272 | params rms: 0.060575 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:30:16.283746 | finish at 2026-10-17 18:03:43 + [2026-10-14 03:37:15] iteration 982/ 2384 | consumed samples: 31424 | elapsed time per iteration (ms): 228056.3 | throughput per GPU (TFLOP/s/GPU): 26.3 | MFU 8.43% | FLOPs estimator: workload-aware | logical tokens/s: 73565.9 | valid tokens/s: 73565.9 | physical tokens/s: 73948.6 | loss tokens/s: 53467.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.762619E-01 | lm-global-supervised-tokens: 1.219370E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.588969E-02 | lm-token-buffer-mib: 2.329735E+02 | global_load_balancing_loss: 9.993302E-01 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11187.230 | params rms: 0.060575 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:48:54.970601 | finish at 2026-10-17 20:26:09 + [2026-10-14 03:40:56] iteration 983/ 2384 | consumed samples: 31456 | elapsed time per iteration (ms): 221776.8 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.13% | FLOPs estimator: workload-aware | logical tokens/s: 75648.9 | valid tokens/s: 75648.9 | physical tokens/s: 76020.2 | loss tokens/s: 59486.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.524359E-01 | lm-global-supervised-tokens: 1.319282E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 6.202398E-02 | lm-token-buffer-mib: 2.325682E+02 | global_load_balancing_loss: 9.988551E-01 | loss scale: 1.0 | grad norm: 0.230 | params norm: 11187.186 | params rms: 0.060575 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:18:29.255599 | finish at 2026-10-17 17:59:26 + [2026-10-14 03:44:44] iteration 984/ 2384 | consumed samples: 31488 | elapsed time per iteration (ms): 228180.7 | throughput per GPU (TFLOP/s/GPU): 25.1 | MFU 8.04% | FLOPs estimator: workload-aware | logical tokens/s: 73525.9 | valid tokens/s: 73525.9 | physical tokens/s: 73968.9 | loss tokens/s: 55426.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.411375E-01 | lm-global-supervised-tokens: 1.264734E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.957076E-02 | lm-token-buffer-mib: 2.330129E+02 | global_load_balancing_loss: 1.000577E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11187.146 | params rms: 0.060575 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:44:12.967405 | finish at 2026-10-17 20:28:57 + [2026-10-14 03:48:26] iteration 985/ 2384 | consumed samples: 31520 | elapsed time per iteration (ms): 221288.1 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.79% | FLOPs estimator: workload-aware | logical tokens/s: 75816.0 | valid tokens/s: 75816.0 | physical tokens/s: 76515.5 | loss tokens/s: 59008.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.187114E-01 | lm-global-supervised-tokens: 1.305780E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.632472E-02 | lm-token-buffer-mib: 2.350279E+02 | global_load_balancing_loss: 9.998204E-01 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11187.113 | params rms: 0.060575 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:59:42.121284 | finish at 2026-10-17 17:48:08 + [2026-10-14 03:52:06] iteration 986/ 2384 | consumed samples: 31552 | elapsed time per iteration (ms): 220534.5 | throughput per GPU (TFLOP/s/GPU): 19.6 | MFU 6.27% | FLOPs estimator: workload-aware | logical tokens/s: 76075.2 | valid tokens/s: 76075.2 | physical tokens/s: 76642.6 | loss tokens/s: 56433.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.668460E-01 | lm-global-supervised-tokens: 1.244560E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.661248E-02 | lm-token-buffer-mib: 2.329836E+02 | global_load_balancing_loss: 1.001741E+00 | loss scale: 1.0 | grad norm: 0.112 | params norm: 11187.082 | params rms: 0.060574 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:38:27.166175 | finish at 2026-10-17 17:30:33 + [2026-10-14 03:55:50] iteration 987/ 2384 | consumed samples: 31584 | elapsed time per iteration (ms): 223797.9 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.42% | FLOPs estimator: workload-aware | logical tokens/s: 74965.8 | valid tokens/s: 74965.8 | physical tokens/s: 76457.3 | loss tokens/s: 58758.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.339434E-01 | lm-global-supervised-tokens: 1.315002E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.391366E-02 | lm-token-buffer-mib: 2.355314E+02 | global_load_balancing_loss: 1.000952E+00 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11187.052 | params rms: 0.060574 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:50:45.714875 | finish at 2026-10-17 18:46:36 + [2026-10-14 03:59:33] iteration 988/ 2384 | consumed samples: 31616 | elapsed time per iteration (ms): 223326.7 | throughput per GPU (TFLOP/s/GPU): 25.8 | MFU 8.27% | FLOPs estimator: workload-aware | logical tokens/s: 75124.0 | valid tokens/s: 75124.0 | physical tokens/s: 75997.3 | loss tokens/s: 55361.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.946305E-01 | lm-global-supervised-tokens: 1.236359E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.812280E-02 | lm-token-buffer-mib: 2.340161E+02 | global_load_balancing_loss: 1.001910E+00 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11187.025 | params rms: 0.060574 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:36:04.009923 | finish at 2026-10-17 18:35:37 + [2026-10-14 04:03:15] iteration 989/ 2384 | consumed samples: 31648 | elapsed time per iteration (ms): 221497.9 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.98% | FLOPs estimator: workload-aware | logical tokens/s: 75744.3 | valid tokens/s: 75744.3 | physical tokens/s: 76540.7 | loss tokens/s: 60626.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.990389E-01 | lm-global-supervised-tokens: 1.342866E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.208784E-02 | lm-token-buffer-mib: 2.342412E+02 | global_load_balancing_loss: 9.997748E-01 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11186.998 | params rms: 0.060574 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:49:49.615080 | finish at 2026-10-17 17:53:05 + [2026-10-14 04:07:18] iteration 990/ 2384 | consumed samples: 31680 | elapsed time per iteration (ms): 243475.6 | throughput per GPU (TFLOP/s/GPU): 18.4 | MFU 5.91% | FLOPs estimator: workload-aware | logical tokens/s: 68907.1 | valid tokens/s: 68907.1 | physical tokens/s: 69652.6 | loss tokens/s: 53353.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.568887E-01 | lm-global-supervised-tokens: 1.299038E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 6.011655E-02 | lm-token-buffer-mib: 2.333877E+02 | global_load_balancing_loss: 9.996936E-01 | loss scale: 1.0 | grad norm: 0.169 | params norm: 11186.968 | params rms: 0.060574 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:16:45.044236 | finish at 2026-10-18 02:24:03 + [2026-10-14 04:10:59] iteration 991/ 2384 | consumed samples: 31712 | elapsed time per iteration (ms): 220215.1 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.98% | FLOPs estimator: workload-aware | logical tokens/s: 76185.6 | valid tokens/s: 76185.6 | physical tokens/s: 76903.4 | loss tokens/s: 58041.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.236853E-01 | lm-global-supervised-tokens: 1.278155E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.339634E-02 | lm-token-buffer-mib: 2.333896E+02 | global_load_balancing_loss: 1.000455E+00 | loss scale: 1.0 | grad norm: 0.109 | params norm: 11186.938 | params rms: 0.060574 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:12:39.617100 | finish at 2026-10-17 17:23:38 + [2026-10-14 04:14:40] iteration 992/ 2384 | consumed samples: 31744 | elapsed time per iteration (ms): 221729.2 | throughput per GPU (TFLOP/s/GPU): 23.2 | MFU 7.44% | FLOPs estimator: workload-aware | logical tokens/s: 75665.1 | valid tokens/s: 75665.1 | physical tokens/s: 76176.5 | loss tokens/s: 55005.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.497244E-01 | lm-global-supervised-tokens: 1.219642E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.799822E-02 | lm-token-buffer-mib: 2.334988E+02 | global_load_balancing_loss: 9.986709E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11186.910 | params rms: 0.060573 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:44:06.994137 | finish at 2026-10-17 17:58:47 + [2026-10-14 04:18:19] iteration 993/ 2384 | consumed samples: 31776 | elapsed time per iteration (ms): 218604.1 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.25% | FLOPs estimator: workload-aware | logical tokens/s: 76746.8 | valid tokens/s: 76746.8 | physical tokens/s: 76928.0 | loss tokens/s: 53190.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.430862E-01 | lm-global-supervised-tokens: 1.162773E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.247158E-02 | lm-token-buffer-mib: 2.322444E+02 | global_load_balancing_loss: 9.989948E-01 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11186.881 | params rms: 0.060573 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:27:58.335254 | finish at 2026-10-17 16:46:17 + [2026-10-14 04:22:02] iteration 994/ 2384 | consumed samples: 31808 | elapsed time per iteration (ms): 222820.8 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.89% | FLOPs estimator: workload-aware | logical tokens/s: 75294.6 | valid tokens/s: 75294.6 | physical tokens/s: 76038.0 | loss tokens/s: 57466.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.635076E-01 | lm-global-supervised-tokens: 1.280481E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.504499E-02 | lm-token-buffer-mib: 2.355837E+02 | global_load_balancing_loss: 1.002524E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11186.849 | params rms: 0.060573 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:02:00.861387 | finish at 2026-10-17 18:24:03 + [2026-10-14 04:25:45] iteration 995/ 2384 | consumed samples: 31840 | elapsed time per iteration (ms): 223578.3 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.05% | FLOPs estimator: workload-aware | logical tokens/s: 75039.4 | valid tokens/s: 75039.4 | physical tokens/s: 75761.8 | loss tokens/s: 51810.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.210961E-01 | lm-global-supervised-tokens: 1.158369E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.053175E-02 | lm-token-buffer-mib: 2.342620E+02 | global_load_balancing_loss: 1.001479E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11186.815 | params rms: 0.060573 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:15:50.216310 | finish at 2026-10-17 18:41:36 + [2026-10-14 04:29:31] iteration 996/ 2384 | consumed samples: 31872 | elapsed time per iteration (ms): 225303.6 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.37% | FLOPs estimator: workload-aware | logical tokens/s: 74464.7 | valid tokens/s: 74464.7 | physical tokens/s: 75115.7 | loss tokens/s: 58541.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.756017E-01 | lm-global-supervised-tokens: 1.318955E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 5.450563E-02 | lm-token-buffer-mib: 2.333459E+02 | global_load_balancing_loss: 1.001698E+00 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11186.783 | params rms: 0.060573 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:52:01.419735 | finish at 2026-10-17 19:21:32 + [2026-10-14 04:33:14] iteration 997/ 2384 | consumed samples: 31904 | elapsed time per iteration (ms): 223356.7 | throughput per GPU (TFLOP/s/GPU): 24.0 | MFU 7.70% | FLOPs estimator: workload-aware | logical tokens/s: 75113.7 | valid tokens/s: 75113.7 | physical tokens/s: 75449.5 | loss tokens/s: 56514.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.213450E-01 | lm-global-supervised-tokens: 1.262278E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.251522E-02 | lm-token-buffer-mib: 2.325446E+02 | global_load_balancing_loss: 1.000000E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11186.753 | params rms: 0.060573 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:03:15.789448 | finish at 2026-10-17 18:36:30 + [2026-10-14 04:36:56] iteration 998/ 2384 | consumed samples: 31936 | elapsed time per iteration (ms): 222103.9 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.87% | FLOPs estimator: workload-aware | logical tokens/s: 75537.6 | valid tokens/s: 75537.6 | physical tokens/s: 76020.6 | loss tokens/s: 57678.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.432881E-01 | lm-global-supervised-tokens: 1.281064E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.585545E-02 | lm-token-buffer-mib: 2.329238E+02 | global_load_balancing_loss: 1.000805E+00 | loss scale: 1.0 | grad norm: 0.134 | params norm: 11186.722 | params rms: 0.060572 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:30:35.979233 | finish at 2026-10-17 18:07:32 + [2026-10-14 04:40:42] iteration 999/ 2384 | consumed samples: 31968 | elapsed time per iteration (ms): 225476.3 | throughput per GPU (TFLOP/s/GPU): 19.9 | MFU 6.39% | FLOPs estimator: workload-aware | logical tokens/s: 74407.7 | valid tokens/s: 74407.7 | physical tokens/s: 75176.2 | loss tokens/s: 57757.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.520673E-01 | lm-global-supervised-tokens: 1.302294E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.508743E-02 | lm-token-buffer-mib: 2.351771E+02 | global_load_balancing_loss: 9.992158E-01 | loss scale: 1.0 | grad norm: 0.119 | params norm: 11186.694 | params rms: 0.060572 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:44:44.704560 | finish at 2026-10-17 19:25:26 + [2026-10-14 04:44:43] iteration 1000/ 2384 | consumed samples: 32000 | elapsed time per iteration (ms): 241523.7 | throughput per GPU (TFLOP/s/GPU): 17.1 | MFU 5.48% | FLOPs estimator: workload-aware | logical tokens/s: 69464.0 | valid tokens/s: 69464.0 | physical tokens/s: 70043.6 | loss tokens/s: 52571.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.863227E-01 | lm-global-supervised-tokens: 1.269728E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.250993E-02 | lm-token-buffer-mib: 2.341956E+02 | global_load_balancing_loss: 1.000100E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11186.664 | params rms: 0.060572 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:51:08.865143 | finish at 2026-10-18 01:35:52 +times across ranks (ms): + save-checkpoint: + rank 0: 24809.33 + rank 1: 24809.36 + rank 2: 24809.36 + rank 3: 24809.35 + rank 4: 24809.34 + rank 5: 24809.37 + rank 6: 24809.37 + rank 7: 24809.34 + rank 8: 24809.11 + rank 9: 24809.14 + rank 10: 24809.10 + rank 11: 24809.10 + rank 12: 24809.12 + rank 13: 24809.12 + rank 14: 24809.11 + rank 15: 24809.15 + rank 16: 24810.18 + rank 17: 24810.25 + rank 18: 24810.19 + rank 19: 24810.17 + rank 20: 24810.13 + rank 21: 24810.15 + rank 22: 24810.13 + rank 23: 24810.12 + rank 24: 24809.35 + rank 25: 24809.40 + rank 26: 24809.48 + rank 27: 24809.82 + rank 28: 24809.34 + rank 29: 24809.38 + rank 30: 24809.32 + rank 31: 24809.36 + rank 32: 24809.21 + rank 33: 24809.26 + rank 34: 24809.31 + rank 35: 24809.24 + rank 36: 24809.19 + rank 37: 24809.17 + rank 38: 24809.34 + rank 39: 24809.12 + rank 40: 24809.09 + rank 41: 24809.09 + rank 42: 24809.07 + rank 43: 24808.93 + rank 44: 24809.10 + rank 45: 24809.05 + rank 46: 24809.05 + rank 47: 24809.04 + rank 48: 24808.49 + rank 49: 24808.57 + rank 50: 24808.51 + rank 51: 24808.48 + rank 52: 24808.54 + rank 53: 24808.61 + rank 54: 24808.57 + rank 55: 24808.52 + rank 56: 24809.17 + rank 57: 24809.20 + rank 58: 24809.26 + rank 59: 24809.19 + rank 60: 24809.21 + rank 61: 24809.16 + rank 62: 24809.18 + rank 63: 24809.17 + [2026-10-14 04:49:14] iteration 1001/ 2384 | consumed samples: 32032 | elapsed time per iteration (ms): 245913.9 | throughput per GPU (TFLOP/s/GPU): 19.5 | MFU 6.27% | FLOPs estimator: workload-aware | logical tokens/s: 68223.7 | valid tokens/s: 68223.7 | physical tokens/s: 69085.9 | loss tokens/s: 53478.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.105592E-01 | lm-global-supervised-tokens: 1.315110E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 7.470369E-02 | lm-token-buffer-mib: 2.355911E+02 | global_load_balancing_loss: 1.000921E+00 | loss scale: 1.0 | grad norm: 0.148 | params norm: 11186.635 | params rms: 0.060572 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:28:18.883332 | finish at 2026-10-18 03:17:33 +times across ranks (ms): + forward-backward: + rank 0: 245088.81 + rank 1: 245090.97 + rank 2: 245090.23 + rank 3: 245094.62 + rank 4: 245095.08 + rank 5: 245096.28 + rank 6: 245089.52 + rank 7: 245091.20 + rank 8: 245636.19 + rank 9: 245647.88 + rank 10: 245643.39 + rank 11: 245644.42 + rank 12: 245648.38 + rank 13: 245648.03 + rank 14: 245636.81 + rank 15: 245644.41 + rank 16: 245629.42 + rank 17: 245634.48 + rank 18: 245633.52 + rank 19: 245636.78 + rank 20: 245640.61 + rank 21: 245640.89 + rank 22: 245635.42 + rank 23: 245630.80 + rank 24: 245633.84 + rank 25: 245644.59 + rank 26: 245642.81 + rank 27: 245640.61 + rank 28: 245643.58 + rank 29: 245639.89 + rank 30: 245641.02 + rank 31: 245636.69 + rank 32: 245074.53 + rank 33: 245075.23 + rank 34: 245073.02 + rank 35: 245076.09 + rank 36: 245076.36 + rank 37: 245077.42 + rank 38: 245074.36 + rank 39: 245075.56 + rank 40: 245628.39 + rank 41: 245638.88 + rank 42: 245633.80 + rank 43: 245633.11 + rank 44: 245636.14 + rank 45: 245635.50 + rank 46: 245631.88 + rank 47: 245638.30 + rank 48: 245625.41 + rank 49: 245628.89 + rank 50: 245628.11 + rank 51: 245630.06 + rank 52: 245633.41 + rank 53: 245632.69 + rank 54: 245633.80 + rank 55: 245627.62 + rank 56: 245645.97 + rank 57: 245652.23 + rank 58: 245651.98 + rank 59: 245649.11 + rank 60: 245649.77 + rank 61: 245645.31 + rank 62: 245653.45 + rank 63: 245648.86 + forward-compute: + rank 0: 146876.77 + rank 1: 146879.17 + rank 2: 146874.33 + rank 3: 146875.45 + rank 4: 146869.38 + rank 5: 146878.28 + rank 6: 146879.22 + rank 7: 146881.17 + rank 8: 147512.06 + rank 9: 147524.08 + rank 10: 147523.00 + rank 11: 147517.88 + rank 12: 147521.17 + rank 13: 147521.50 + rank 14: 147518.17 + rank 15: 147526.20 + rank 16: 147413.48 + rank 17: 147417.59 + rank 18: 147416.97 + rank 19: 147418.23 + rank 20: 147419.09 + rank 21: 147417.69 + rank 22: 147418.41 + rank 23: 147420.62 + rank 24: 147430.02 + rank 25: 147436.52 + rank 26: 147434.97 + rank 27: 147433.64 + rank 28: 147431.72 + rank 29: 147427.28 + rank 30: 147433.38 + rank 31: 147427.78 + rank 32: 59027.99 + rank 33: 59048.03 + rank 34: 59054.38 + rank 35: 59064.59 + rank 36: 59064.62 + rank 37: 59078.88 + rank 38: 59060.59 + rank 39: 59061.25 + rank 40: 59650.97 + rank 41: 59637.23 + rank 42: 59630.93 + rank 43: 59636.82 + rank 44: 59623.75 + rank 45: 59652.07 + rank 46: 59603.73 + rank 47: 59643.73 + rank 48: 59597.16 + rank 49: 59597.99 + rank 50: 59599.89 + rank 51: 59589.55 + rank 52: 59650.96 + rank 53: 59653.60 + rank 54: 59594.23 + rank 55: 59606.91 + rank 56: 59613.82 + rank 57: 59617.97 + rank 58: 59600.69 + rank 59: 59605.55 + rank 60: 59615.42 + rank 61: 59621.52 + rank 62: 59601.17 + rank 63: 59654.89 + backward-compute: + rank 0: 98159.09 + rank 1: 98154.92 + rank 2: 98163.74 + rank 3: 98165.16 + rank 4: 98165.29 + rank 5: 98166.77 + rank 6: 98159.98 + rank 7: 98156.45 + rank 8: 98070.36 + rank 9: 98070.18 + rank 10: 98065.95 + rank 11: 98071.47 + rank 12: 98070.33 + rank 13: 98071.48 + rank 14: 98064.14 + rank 15: 98063.28 + rank 16: 98169.50 + rank 17: 98160.91 + rank 18: 98160.03 + rank 19: 98163.34 + rank 20: 98167.55 + rank 21: 98168.22 + rank 22: 98164.53 + rank 23: 98158.65 + rank 24: 98145.09 + rank 25: 98143.65 + rank 26: 98144.03 + rank 27: 98142.79 + rank 28: 98143.62 + rank 29: 98150.03 + rank 30: 98145.93 + rank 31: 98137.26 + rank 32: 186006.14 + rank 33: 185987.55 + rank 34: 185979.72 + rank 35: 185973.14 + rank 36: 185974.14 + rank 37: 185960.92 + rank 38: 185974.70 + rank 39: 185977.00 + rank 40: 185942.22 + rank 41: 185967.78 + rank 42: 185969.62 + rank 43: 185961.88 + rank 44: 185978.47 + rank 45: 185948.81 + rank 46: 185992.23 + rank 47: 185959.05 + rank 48: 185990.00 + rank 49: 185992.25 + rank 50: 185984.98 + rank 51: 186003.53 + rank 52: 185942.14 + rank 53: 185940.39 + rank 54: 185998.91 + rank 55: 185982.70 + rank 56: 185999.33 + rank 57: 185999.06 + rank 58: 186015.41 + rank 59: 186009.19 + rank 60: 185999.62 + rank 61: 185986.30 + rank 62: 186018.02 + rank 63: 185959.48 + batch-generator: + rank 0: 36.86 + rank 1: 35.94 + rank 2: 37.84 + rank 3: 35.96 + rank 4: 38.52 + rank 5: 37.83 + rank 6: 37.51 + rank 7: 36.71 + rank 8: 69.31 + rank 9: 70.46 + rank 10: 70.08 + rank 11: 69.82 + rank 12: 70.68 + rank 13: 71.15 + rank 14: 71.58 + rank 15: 71.09 + rank 16: 33.97 + rank 17: 37.54 + rank 18: 37.59 + rank 19: 30.21 + rank 20: 36.50 + rank 21: 37.28 + rank 22: 37.81 + rank 23: 35.99 + rank 24: 39.00 + rank 25: 45.84 + rank 26: 50.75 + rank 27: 50.91 + rank 28: 51.53 + rank 29: 49.59 + rank 30: 46.41 + rank 31: 46.27 + rank 32: 48.71 + rank 33: 51.94 + rank 34: 48.85 + rank 35: 42.42 + rank 36: 41.07 + rank 37: 42.45 + rank 38: 45.99 + rank 39: 41.78 + rank 40: 37.33 + rank 41: 36.63 + rank 42: 35.17 + rank 43: 37.95 + rank 44: 36.23 + rank 45: 35.85 + rank 46: 38.53 + rank 47: 32.65 + rank 48: 56.03 + rank 49: 55.29 + rank 50: 62.33 + rank 51: 49.49 + rank 52: 53.03 + rank 53: 52.92 + rank 54: 39.23 + rank 55: 52.78 + rank 56: 35.19 + rank 57: 38.35 + rank 58: 38.70 + rank 59: 38.20 + rank 60: 37.72 + rank 61: 37.18 + rank 62: 27.93 + rank 63: 33.63 + embedding-grads-all-reduce: + rank 0: 0.03 + rank 1: 0.03 + rank 2: 0.03 + rank 3: 0.03 + rank 4: 0.03 + rank 5: 0.03 + rank 6: 0.03 + rank 7: 0.03 + rank 8: 0.03 + rank 9: 0.03 + rank 10: 0.03 + rank 11: 0.03 + rank 12: 0.03 + rank 13: 0.03 + rank 14: 0.02 + rank 15: 0.03 + rank 16: 0.03 + rank 17: 0.03 + rank 18: 0.03 + rank 19: 0.03 + rank 20: 0.03 + rank 21: 0.03 + rank 22: 0.03 + rank 23: 0.03 + rank 24: 0.04 + rank 25: 0.04 + rank 26: 0.04 + rank 27: 0.04 + rank 28: 0.04 + rank 29: 0.04 + rank 30: 0.04 + rank 31: 0.04 + rank 32: 0.03 + rank 33: 0.03 + rank 34: 0.04 + rank 35: 0.04 + rank 36: 0.03 + rank 37: 0.04 + rank 38: 0.03 + rank 39: 0.04 + rank 40: 0.03 + rank 41: 0.03 + rank 42: 0.03 + rank 43: 0.03 + rank 44: 0.03 + rank 45: 0.03 + rank 46: 0.03 + rank 47: 0.02 + rank 48: 0.04 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.03 + rank 57: 0.03 + rank 58: 0.03 + rank 59: 0.03 + rank 60: 0.03 + rank 61: 0.03 + rank 62: 0.03 + rank 63: 0.03 + all-grads-sync: + rank 0: 1.07 + rank 1: 0.85 + rank 2: 0.90 + rank 3: 1.28 + rank 4: 0.90 + rank 5: 0.84 + rank 6: 0.93 + rank 7: 0.80 + rank 8: 1.31 + rank 9: 1.05 + rank 10: 0.85 + rank 11: 1.25 + rank 12: 1.21 + rank 13: 0.83 + rank 14: 0.86 + rank 15: 1.18 + rank 16: 1.43 + rank 17: 0.80 + rank 18: 0.99 + rank 19: 0.91 + rank 20: 0.86 + rank 21: 1.24 + rank 22: 0.83 + rank 23: 1.16 + rank 24: 1.18 + rank 25: 1.48 + rank 26: 1.47 + rank 27: 1.39 + rank 28: 1.74 + rank 29: 1.81 + rank 30: 1.68 + rank 31: 1.33 + rank 32: 1.25 + rank 33: 1.31 + rank 34: 1.50 + rank 35: 1.34 + rank 36: 1.27 + rank 37: 1.29 + rank 38: 1.24 + rank 39: 1.29 + rank 40: 0.84 + rank 41: 0.82 + rank 42: 0.78 + rank 43: 0.79 + rank 44: 0.86 + rank 45: 0.88 + rank 46: 1.20 + rank 47: 0.86 + rank 48: 1.09 + rank 49: 1.02 + rank 50: 1.07 + rank 51: 0.81 + rank 52: 1.05 + rank 53: 0.86 + rank 54: 0.94 + rank 55: 1.28 + rank 56: 1.02 + rank 57: 0.92 + rank 58: 1.13 + rank 59: 0.83 + rank 60: 1.05 + rank 61: 0.94 + rank 62: 0.83 + rank 63: 0.83 + params-all-gather: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.02 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.02 + rank 17: 0.02 + rank 18: 0.03 + rank 19: 0.02 + rank 20: 0.02 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.02 + rank 24: 0.04 + rank 25: 0.03 + rank 26: 0.04 + rank 27: 0.03 + rank 28: 0.03 + rank 29: 0.04 + rank 30: 0.04 + rank 31: 0.03 + rank 32: 0.03 + rank 33: 0.03 + rank 34: 0.02 + rank 35: 0.02 + rank 36: 0.03 + rank 37: 0.03 + rank 38: 0.03 + rank 39: 0.03 + rank 40: 0.02 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.02 + rank 45: 0.02 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.02 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.02 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.02 + optimizer-copy-to-main-grad: + rank 0: 2.27 + rank 1: 1.88 + rank 2: 1.82 + rank 3: 2.26 + rank 4: 1.77 + rank 5: 1.93 + rank 6: 2.02 + rank 7: 1.95 + rank 8: 1.85 + rank 9: 2.20 + rank 10: 1.80 + rank 11: 1.76 + rank 12: 1.99 + rank 13: 1.82 + rank 14: 1.96 + rank 15: 2.25 + rank 16: 2.58 + rank 17: 2.38 + rank 18: 2.32 + rank 19: 2.29 + rank 20: 2.29 + rank 21: 2.41 + rank 22: 2.68 + rank 23: 2.49 + rank 24: 2.00 + rank 25: 2.35 + rank 26: 1.98 + rank 27: 2.18 + rank 28: 2.54 + rank 29: 2.25 + rank 30: 2.17 + rank 31: 2.26 + rank 32: 2.11 + rank 33: 2.03 + rank 34: 2.09 + rank 35: 2.33 + rank 36: 2.15 + rank 37: 2.26 + rank 38: 2.32 + rank 39: 1.96 + rank 40: 1.83 + rank 41: 1.89 + rank 42: 1.97 + rank 43: 1.94 + rank 44: 1.88 + rank 45: 2.12 + rank 46: 1.77 + rank 47: 1.79 + rank 48: 1.93 + rank 49: 1.80 + rank 50: 2.48 + rank 51: 1.73 + rank 52: 2.03 + rank 53: 1.76 + rank 54: 1.80 + rank 55: 1.82 + rank 56: 1.81 + rank 57: 2.50 + rank 58: 1.82 + rank 59: 1.76 + rank 60: 1.81 + rank 61: 1.76 + rank 62: 1.83 + rank 63: 1.76 + optimizer-inner-step: + rank 0: 51.19 + rank 1: 51.18 + rank 2: 50.94 + rank 3: 51.10 + rank 4: 50.87 + rank 5: 50.97 + rank 6: 51.35 + rank 7: 51.32 + rank 8: 51.14 + rank 9: 51.14 + rank 10: 50.97 + rank 11: 51.13 + rank 12: 51.15 + rank 13: 51.39 + rank 14: 51.10 + rank 15: 50.94 + rank 16: 51.50 + rank 17: 51.17 + rank 18: 51.28 + rank 19: 51.22 + rank 20: 51.12 + rank 21: 51.08 + rank 22: 51.71 + rank 23: 51.18 + rank 24: 50.50 + rank 25: 51.04 + rank 26: 50.99 + rank 27: 50.72 + rank 28: 51.44 + rank 29: 50.80 + rank 30: 50.97 + rank 31: 51.03 + rank 32: 51.19 + rank 33: 51.07 + rank 34: 51.12 + rank 35: 51.47 + rank 36: 51.27 + rank 37: 51.10 + rank 38: 51.02 + rank 39: 51.02 + rank 40: 51.09 + rank 41: 51.21 + rank 42: 51.22 + rank 43: 51.31 + rank 44: 50.90 + rank 45: 51.56 + rank 46: 51.14 + rank 47: 50.85 + rank 48: 48.48 + rank 49: 48.41 + rank 50: 60.25 + rank 51: 48.24 + rank 52: 48.52 + rank 53: 48.39 + rank 54: 48.59 + rank 55: 48.48 + rank 56: 50.72 + rank 57: 51.93 + rank 58: 51.01 + rank 59: 50.90 + rank 60: 51.24 + rank 61: 50.91 + rank 62: 50.95 + rank 63: 50.88 + optimizer-copy-main-to-model-params: + rank 0: 4.38 + rank 1: 3.82 + rank 2: 3.85 + rank 3: 4.42 + rank 4: 3.72 + rank 5: 4.01 + rank 6: 4.09 + rank 7: 4.04 + rank 8: 3.89 + rank 9: 4.42 + rank 10: 3.86 + rank 11: 3.84 + rank 12: 3.97 + rank 13: 3.87 + rank 14: 4.09 + rank 15: 4.26 + rank 16: 4.43 + rank 17: 4.06 + rank 18: 4.43 + rank 19: 4.02 + rank 20: 4.18 + rank 21: 4.34 + rank 22: 4.79 + rank 23: 4.33 + rank 24: 4.02 + rank 25: 4.63 + rank 26: 4.00 + rank 27: 4.14 + rank 28: 4.49 + rank 29: 4.43 + rank 30: 4.37 + rank 31: 4.14 + rank 32: 4.16 + rank 33: 3.96 + rank 34: 4.01 + rank 35: 4.59 + rank 36: 4.26 + rank 37: 4.17 + rank 38: 4.49 + rank 39: 4.21 + rank 40: 3.83 + rank 41: 3.97 + rank 42: 3.95 + rank 43: 4.05 + rank 44: 3.96 + rank 45: 4.29 + rank 46: 3.74 + rank 47: 3.73 + rank 48: 4.05 + rank 49: 3.92 + rank 50: 5.03 + rank 51: 3.96 + rank 52: 4.19 + rank 53: 3.91 + rank 54: 3.94 + rank 55: 3.92 + rank 56: 3.73 + rank 57: 4.79 + rank 58: 3.94 + rank 59: 3.74 + rank 60: 3.74 + rank 61: 3.69 + rank 62: 3.89 + rank 63: 3.68 + optimizer: + rank 0: 66.31 + rank 1: 65.43 + rank 2: 65.16 + rank 3: 66.13 + rank 4: 64.96 + rank 5: 65.48 + rank 6: 65.96 + rank 7: 65.88 + rank 8: 65.50 + rank 9: 66.22 + rank 10: 65.26 + rank 11: 65.38 + rank 12: 65.60 + rank 13: 65.64 + rank 14: 65.65 + rank 15: 65.78 + rank 16: 66.79 + rank 17: 65.98 + rank 18: 66.44 + rank 19: 65.92 + rank 20: 66.02 + rank 21: 66.23 + rank 22: 67.43 + rank 23: 66.47 + rank 24: 65.31 + rank 25: 66.37 + rank 26: 65.56 + rank 27: 65.81 + rank 28: 66.59 + rank 29: 65.97 + rank 30: 66.01 + rank 31: 65.86 + rank 32: 65.80 + rank 33: 65.38 + rank 34: 65.52 + rank 35: 66.71 + rank 36: 66.05 + rank 37: 65.72 + rank 38: 66.10 + rank 39: 65.65 + rank 40: 65.28 + rank 41: 65.64 + rank 42: 65.62 + rank 43: 65.83 + rank 44: 65.34 + rank 45: 66.41 + rank 46: 65.25 + rank 47: 64.97 + rank 48: 65.67 + rank 49: 65.36 + rank 50: 78.72 + rank 51: 65.22 + rank 52: 65.91 + rank 53: 65.34 + rank 54: 65.56 + rank 55: 65.43 + rank 56: 64.90 + rank 57: 67.60 + rank 58: 65.44 + rank 59: 65.01 + rank 60: 65.38 + rank 61: 65.02 + rank 62: 65.30 + rank 63: 64.95 + interval-time: + rank 0: 0.97 + rank 1: 0.94 + rank 2: 0.95 + rank 3: 1.84 + rank 4: 0.89 + rank 5: 1.00 + rank 6: 0.96 + rank 7: 1.01 + rank 8: 0.83 + rank 9: 0.85 + rank 10: 0.86 + rank 11: 0.83 + rank 12: 0.84 + rank 13: 0.73 + rank 14: 0.85 + rank 15: 0.85 + rank 16: 0.86 + rank 17: 0.85 + rank 18: 0.87 + rank 19: 0.88 + rank 20: 0.87 + rank 21: 0.86 + rank 22: 0.77 + rank 23: 0.77 + rank 24: 0.97 + rank 25: 0.98 + rank 26: 0.86 + rank 27: 0.95 + rank 28: 0.95 + rank 29: 0.94 + rank 30: 0.89 + rank 31: 1.09 + rank 32: 0.96 + rank 33: 0.98 + rank 34: 0.96 + rank 35: 0.86 + rank 36: 0.81 + rank 37: 0.85 + rank 38: 0.82 + rank 39: 0.84 + rank 40: 0.85 + rank 41: 0.84 + rank 42: 0.82 + rank 43: 0.76 + rank 44: 0.74 + rank 45: 0.83 + rank 46: 0.83 + rank 47: 0.82 + rank 48: 9.21 + rank 49: 9.19 + rank 50: 9.20 + rank 51: 9.18 + rank 52: 9.20 + rank 53: 9.20 + rank 54: 9.21 + rank 55: 9.18 + rank 56: 0.97 + rank 57: 0.89 + rank 58: 0.91 + rank 59: 0.87 + rank 60: 0.92 + rank 61: 0.93 + rank 62: 0.94 + rank 63: 1.46 + [2026-10-14 04:53:03] iteration 1002/ 2384 | consumed samples: 32064 | elapsed time per iteration (ms): 229610.4 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.56% | FLOPs estimator: workload-aware | logical tokens/s: 73068.1 | valid tokens/s: 73068.1 | physical tokens/s: 73773.7 | loss tokens/s: 52419.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.524406E-01 | lm-global-supervised-tokens: 1.203612E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.713766E-02 | lm-token-buffer-mib: 2.340956E+02 | global_load_balancing_loss: 9.993061E-01 | loss scale: 1.0 | grad norm: 0.251 | params norm: 11186.606 | params rms: 0.060572 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:08:41.518380 | finish at 2026-10-17 21:01:45 +times across ranks (ms): + forward-backward: + rank 0: 223245.02 + rank 1: 223326.61 + rank 2: 223321.97 + rank 3: 223324.11 + rank 4: 223326.61 + rank 5: 223322.83 + rank 6: 223325.45 + rank 7: 223324.05 + rank 8: 223326.08 + rank 9: 223324.72 + rank 10: 223325.05 + rank 11: 223323.31 + rank 12: 223324.69 + rank 13: 223326.72 + rank 14: 223328.00 + rank 15: 223326.91 + rank 16: 223327.59 + rank 17: 223327.56 + rank 18: 223328.34 + rank 19: 223327.92 + rank 20: 223325.23 + rank 21: 223327.16 + rank 22: 223324.61 + rank 23: 223326.16 + rank 24: 223326.17 + rank 25: 223318.86 + rank 26: 223315.31 + rank 27: 223325.30 + rank 28: 223322.30 + rank 29: 223325.44 + rank 30: 223320.61 + rank 31: 223326.95 + rank 32: 223246.12 + rank 33: 223329.61 + rank 34: 223322.86 + rank 35: 223323.42 + rank 36: 223325.62 + rank 37: 223322.77 + rank 38: 223324.25 + rank 39: 223327.89 + rank 40: 223328.02 + rank 41: 223327.50 + rank 42: 223326.12 + rank 43: 223324.98 + rank 44: 223326.14 + rank 45: 223326.27 + rank 46: 223326.28 + rank 47: 223330.36 + rank 48: 223327.02 + rank 49: 223326.41 + rank 50: 223325.45 + rank 51: 223325.30 + rank 52: 223322.02 + rank 53: 223323.97 + rank 54: 223324.12 + rank 55: 223322.59 + rank 56: 223328.27 + rank 57: 223322.70 + rank 58: 223319.64 + rank 59: 223329.89 + rank 60: 223326.19 + rank 61: 223329.70 + rank 62: 223326.00 + rank 63: 223328.50 + forward-compute: + rank 0: 114469.48 + rank 1: 114547.39 + rank 2: 114537.41 + rank 3: 114529.35 + rank 4: 114513.04 + rank 5: 114510.02 + rank 6: 114456.94 + rank 7: 114540.48 + rank 8: 114555.74 + rank 9: 114558.68 + rank 10: 114560.91 + rank 11: 114549.68 + rank 12: 114559.51 + rank 13: 114563.79 + rank 14: 114555.02 + rank 15: 114565.85 + rank 16: 114536.91 + rank 17: 114526.16 + rank 18: 114512.90 + rank 19: 114508.60 + rank 20: 114503.02 + rank 21: 114505.57 + rank 22: 114501.65 + rank 23: 114503.42 + rank 24: 114512.36 + rank 25: 114505.42 + rank 26: 114499.51 + rank 27: 114505.58 + rank 28: 114505.05 + rank 29: 114509.40 + rank 30: 114496.32 + rank 31: 114496.20 + rank 32: 62972.00 + rank 33: 63067.85 + rank 34: 63051.09 + rank 35: 63066.29 + rank 36: 63081.55 + rank 37: 63092.05 + rank 38: 63064.27 + rank 39: 63069.20 + rank 40: 63096.31 + rank 41: 63058.11 + rank 42: 63078.89 + rank 43: 63076.16 + rank 44: 63073.60 + rank 45: 63078.25 + rank 46: 63055.07 + rank 47: 63085.56 + rank 48: 63065.04 + rank 49: 63067.31 + rank 50: 63071.41 + rank 51: 63067.24 + rank 52: 63085.92 + rank 53: 63076.70 + rank 54: 63019.71 + rank 55: 63052.31 + rank 56: 63061.11 + rank 57: 63044.35 + rank 58: 63044.84 + rank 59: 63044.70 + rank 60: 63044.55 + rank 61: 63064.52 + rank 62: 63026.84 + rank 63: 63082.06 + backward-compute: + rank 0: 108728.05 + rank 1: 108728.12 + rank 2: 108726.09 + rank 3: 108743.28 + rank 4: 108757.08 + rank 5: 108759.48 + rank 6: 108816.65 + rank 7: 108732.19 + rank 8: 108716.84 + rank 9: 108713.51 + rank 10: 108713.28 + rank 11: 108723.77 + rank 12: 108714.47 + rank 13: 108712.59 + rank 14: 108722.01 + rank 15: 108710.87 + rank 16: 108743.98 + rank 17: 108746.23 + rank 18: 108766.47 + rank 19: 108769.62 + rank 20: 108773.18 + rank 21: 108771.36 + rank 22: 108773.01 + rank 23: 108778.01 + rank 24: 108757.66 + rank 25: 108755.34 + rank 26: 108754.24 + rank 27: 108757.18 + rank 28: 108755.23 + rank 29: 108758.93 + rank 30: 108766.03 + rank 31: 108769.69 + rank 32: 160235.38 + rank 33: 160220.98 + rank 34: 160233.31 + rank 35: 160215.94 + rank 36: 160206.34 + rank 37: 160189.52 + rank 38: 160221.06 + rank 39: 160220.67 + rank 40: 160197.55 + rank 41: 160234.02 + rank 42: 160214.14 + rank 43: 160213.67 + rank 44: 160217.25 + rank 45: 160213.67 + rank 46: 160234.44 + rank 47: 160210.72 + rank 48: 160227.67 + rank 49: 160223.66 + rank 50: 160217.36 + rank 51: 160228.09 + rank 52: 160201.80 + rank 53: 160213.58 + rank 54: 160268.12 + rank 55: 160237.80 + rank 56: 160235.00 + rank 57: 160242.56 + rank 58: 160238.23 + rank 59: 160250.31 + rank 60: 160246.20 + rank 61: 160230.28 + rank 62: 160265.55 + rank 63: 160212.27 + batch-generator: + rank 0: 36.48 + rank 1: 36.99 + rank 2: 36.45 + rank 3: 37.67 + rank 4: 37.51 + rank 5: 38.90 + rank 6: 37.74 + rank 7: 35.86 + rank 8: 55.10 + rank 9: 55.87 + rank 10: 56.96 + rank 11: 55.80 + rank 12: 58.67 + rank 13: 56.23 + rank 14: 54.93 + rank 15: 56.38 + rank 16: 36.48 + rank 17: 37.13 + rank 18: 38.24 + rank 19: 28.73 + rank 20: 37.07 + rank 21: 39.27 + rank 22: 40.16 + rank 23: 36.70 + rank 24: 37.49 + rank 25: 48.01 + rank 26: 51.63 + rank 27: 49.96 + rank 28: 47.93 + rank 29: 50.07 + rank 30: 44.84 + rank 31: 47.97 + rank 32: 44.73 + rank 33: 50.97 + rank 34: 46.86 + rank 35: 40.34 + rank 36: 41.20 + rank 37: 43.09 + rank 38: 43.02 + rank 39: 40.91 + rank 40: 36.44 + rank 41: 37.14 + rank 42: 35.87 + rank 43: 36.61 + rank 44: 33.81 + rank 45: 35.51 + rank 46: 36.60 + rank 47: 32.15 + rank 48: 39.52 + rank 49: 45.70 + rank 50: 45.82 + rank 51: 34.16 + rank 52: 40.89 + rank 53: 41.11 + rank 54: 37.46 + rank 55: 34.35 + rank 56: 34.56 + rank 57: 39.02 + rank 58: 39.86 + rank 59: 37.92 + rank 60: 37.74 + rank 61: 39.90 + rank 62: 28.02 + rank 63: 35.40 + embedding-grads-all-reduce: + rank 0: 0.03 + rank 1: 0.03 + rank 2: 0.03 + rank 3: 0.03 + rank 4: 0.03 + rank 5: 0.03 + rank 6: 0.03 + rank 7: 0.03 + rank 8: 0.03 + rank 9: 0.03 + rank 10: 0.03 + rank 11: 0.03 + rank 12: 0.03 + rank 13: 0.03 + rank 14: 0.02 + rank 15: 0.03 + rank 16: 0.03 + rank 17: 0.03 + rank 18: 0.03 + rank 19: 0.03 + rank 20: 0.03 + rank 21: 0.03 + rank 22: 0.03 + rank 23: 0.03 + rank 24: 0.04 + rank 25: 0.04 + rank 26: 0.04 + rank 27: 0.04 + rank 28: 0.04 + rank 29: 0.04 + rank 30: 0.04 + rank 31: 0.04 + rank 32: 0.03 + rank 33: 0.03 + rank 34: 0.03 + rank 35: 0.04 + rank 36: 0.03 + rank 37: 0.04 + rank 38: 0.04 + rank 39: 0.04 + rank 40: 0.03 + rank 41: 0.02 + rank 42: 0.03 + rank 43: 0.03 + rank 44: 0.03 + rank 45: 0.03 + rank 46: 0.03 + rank 47: 0.04 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.03 + rank 57: 0.03 + rank 58: 0.03 + rank 59: 0.02 + rank 60: 0.03 + rank 61: 0.03 + rank 62: 0.03 + rank 63: 0.03 + all-grads-sync: + rank 0: 1.31 + rank 1: 0.90 + rank 2: 0.81 + rank 3: 1.36 + rank 4: 0.96 + rank 5: 1.15 + rank 6: 0.87 + rank 7: 0.84 + rank 8: 1.18 + rank 9: 1.15 + rank 10: 1.00 + rank 11: 0.81 + rank 12: 0.90 + rank 13: 0.83 + rank 14: 0.96 + rank 15: 0.83 + rank 16: 1.07 + rank 17: 0.86 + rank 18: 1.03 + rank 19: 0.96 + rank 20: 0.91 + rank 21: 1.18 + rank 22: 0.86 + rank 23: 0.83 + rank 24: 1.33 + rank 25: 1.29 + rank 26: 1.67 + rank 27: 1.50 + rank 28: 1.43 + rank 29: 1.47 + rank 30: 1.66 + rank 31: 1.35 + rank 32: 1.62 + rank 33: 1.72 + rank 34: 1.33 + rank 35: 1.50 + rank 36: 1.35 + rank 37: 1.34 + rank 38: 1.46 + rank 39: 1.19 + rank 40: 0.94 + rank 41: 0.95 + rank 42: 0.88 + rank 43: 0.94 + rank 44: 1.55 + rank 45: 0.79 + rank 46: 0.85 + rank 47: 1.50 + rank 48: 1.27 + rank 49: 1.00 + rank 50: 1.02 + rank 51: 0.84 + rank 52: 0.88 + rank 53: 1.31 + rank 54: 0.94 + rank 55: 1.01 + rank 56: 0.87 + rank 57: 0.88 + rank 58: 1.09 + rank 59: 1.20 + rank 60: 0.81 + rank 61: 0.85 + rank 62: 1.18 + rank 63: 1.04 + params-all-gather: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.02 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.02 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.02 + rank 20: 0.03 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.02 + rank 24: 0.03 + rank 25: 0.03 + rank 26: 0.03 + rank 27: 0.03 + rank 28: 0.04 + rank 29: 0.04 + rank 30: 0.03 + rank 31: 0.03 + rank 32: 0.03 + rank 33: 0.03 + rank 34: 0.03 + rank 35: 0.03 + rank 36: 0.02 + rank 37: 0.02 + rank 38: 0.03 + rank 39: 0.03 + rank 40: 0.02 + rank 41: 0.03 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.02 + rank 45: 0.02 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.02 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.02 + optimizer-copy-to-main-grad: + rank 0: 2.26 + rank 1: 1.92 + rank 2: 1.81 + rank 3: 2.23 + rank 4: 1.73 + rank 5: 2.08 + rank 6: 1.99 + rank 7: 2.06 + rank 8: 1.90 + rank 9: 2.20 + rank 10: 1.82 + rank 11: 1.77 + rank 12: 1.87 + rank 13: 1.76 + rank 14: 1.94 + rank 15: 2.10 + rank 16: 2.56 + rank 17: 2.28 + rank 18: 2.32 + rank 19: 2.32 + rank 20: 2.21 + rank 21: 2.39 + rank 22: 2.61 + rank 23: 2.39 + rank 24: 2.11 + rank 25: 2.05 + rank 26: 2.07 + rank 27: 2.27 + rank 28: 2.45 + rank 29: 2.26 + rank 30: 2.12 + rank 31: 2.27 + rank 32: 2.08 + rank 33: 2.04 + rank 34: 1.99 + rank 35: 2.44 + rank 36: 2.12 + rank 37: 2.10 + rank 38: 2.28 + rank 39: 1.98 + rank 40: 1.77 + rank 41: 1.93 + rank 42: 1.84 + rank 43: 2.03 + rank 44: 1.88 + rank 45: 2.06 + rank 46: 1.81 + rank 47: 1.67 + rank 48: 2.00 + rank 49: 1.80 + rank 50: 2.44 + rank 51: 1.74 + rank 52: 1.95 + rank 53: 1.83 + rank 54: 1.81 + rank 55: 1.83 + rank 56: 1.84 + rank 57: 2.45 + rank 58: 1.83 + rank 59: 1.76 + rank 60: 1.76 + rank 61: 1.68 + rank 62: 1.86 + rank 63: 1.71 + optimizer-inner-step: + rank 0: 42.61 + rank 1: 42.57 + rank 2: 42.28 + rank 3: 42.47 + rank 4: 42.23 + rank 5: 42.39 + rank 6: 42.73 + rank 7: 42.72 + rank 8: 42.55 + rank 9: 42.50 + rank 10: 42.32 + rank 11: 42.52 + rank 12: 42.48 + rank 13: 42.73 + rank 14: 42.49 + rank 15: 42.32 + rank 16: 42.97 + rank 17: 42.50 + rank 18: 42.67 + rank 19: 42.54 + rank 20: 42.49 + rank 21: 42.42 + rank 22: 43.10 + rank 23: 42.59 + rank 24: 42.07 + rank 25: 42.54 + rank 26: 42.51 + rank 27: 42.59 + rank 28: 46.28 + rank 29: 42.30 + rank 30: 42.44 + rank 31: 42.62 + rank 32: 42.55 + rank 33: 42.38 + rank 34: 42.52 + rank 35: 49.71 + rank 36: 42.70 + rank 37: 42.55 + rank 38: 42.30 + rank 39: 42.38 + rank 40: 42.41 + rank 41: 42.59 + rank 42: 42.62 + rank 43: 42.70 + rank 44: 42.29 + rank 45: 42.93 + rank 46: 42.59 + rank 47: 42.23 + rank 48: 42.43 + rank 49: 42.35 + rank 50: 54.40 + rank 51: 42.17 + rank 52: 42.48 + rank 53: 42.35 + rank 54: 42.56 + rank 55: 42.52 + rank 56: 42.20 + rank 57: 46.76 + rank 58: 42.44 + rank 59: 42.32 + rank 60: 42.62 + rank 61: 42.30 + rank 62: 42.44 + rank 63: 42.30 + optimizer-copy-main-to-model-params: + rank 0: 4.42 + rank 1: 3.85 + rank 2: 3.76 + rank 3: 4.34 + rank 4: 3.69 + rank 5: 4.02 + rank 6: 4.06 + rank 7: 4.03 + rank 8: 3.84 + rank 9: 4.48 + rank 10: 3.83 + rank 11: 3.80 + rank 12: 3.91 + rank 13: 3.83 + rank 14: 4.01 + rank 15: 4.24 + rank 16: 4.48 + rank 17: 4.08 + rank 18: 4.09 + rank 19: 4.12 + rank 20: 4.10 + rank 21: 4.28 + rank 22: 4.69 + rank 23: 4.25 + rank 24: 4.02 + rank 25: 4.15 + rank 26: 3.99 + rank 27: 4.02 + rank 28: 4.37 + rank 29: 4.39 + rank 30: 4.37 + rank 31: 4.03 + rank 32: 4.13 + rank 33: 3.94 + rank 34: 4.03 + rank 35: 4.55 + rank 36: 4.28 + rank 37: 4.15 + rank 38: 4.48 + rank 39: 3.90 + rank 40: 3.79 + rank 41: 3.98 + rank 42: 3.95 + rank 43: 4.03 + rank 44: 4.00 + rank 45: 4.25 + rank 46: 3.84 + rank 47: 3.72 + rank 48: 4.09 + rank 49: 3.88 + rank 50: 4.96 + rank 51: 3.97 + rank 52: 4.20 + rank 53: 3.86 + rank 54: 3.89 + rank 55: 3.85 + rank 56: 3.78 + rank 57: 4.76 + rank 58: 3.95 + rank 59: 3.76 + rank 60: 3.83 + rank 61: 3.75 + rank 62: 3.89 + rank 63: 3.65 + optimizer: + rank 0: 54.52 + rank 1: 53.64 + rank 2: 53.16 + rank 3: 54.22 + rank 4: 53.03 + rank 5: 53.66 + rank 6: 54.11 + rank 7: 54.04 + rank 8: 53.59 + rank 9: 54.39 + rank 10: 53.34 + rank 11: 53.48 + rank 12: 53.61 + rank 13: 53.73 + rank 14: 53.75 + rank 15: 53.87 + rank 16: 55.13 + rank 17: 54.06 + rank 18: 54.21 + rank 19: 54.13 + rank 20: 54.07 + rank 21: 54.28 + rank 22: 55.42 + rank 23: 54.50 + rank 24: 53.55 + rank 25: 54.01 + rank 26: 53.68 + rank 27: 53.82 + rank 28: 58.03 + rank 29: 54.09 + rank 30: 54.13 + rank 31: 53.89 + rank 32: 53.93 + rank 33: 53.46 + rank 34: 53.66 + rank 35: 61.65 + rank 36: 54.20 + rank 37: 53.90 + rank 38: 54.13 + rank 39: 53.42 + rank 40: 53.36 + rank 41: 53.79 + rank 42: 53.80 + rank 43: 53.92 + rank 44: 53.51 + rank 45: 54.47 + rank 46: 53.58 + rank 47: 53.08 + rank 48: 53.88 + rank 49: 53.45 + rank 50: 67.00 + rank 51: 53.39 + rank 52: 54.03 + rank 53: 53.42 + rank 54: 53.63 + rank 55: 53.55 + rank 56: 53.14 + rank 57: 59.12 + rank 58: 53.60 + rank 59: 53.18 + rank 60: 53.56 + rank 61: 53.16 + rank 62: 53.49 + rank 63: 53.08 + interval-time: + rank 0: 0.83 + rank 1: 0.83 + rank 2: 0.79 + rank 3: 0.76 + rank 4: 0.76 + rank 5: 0.90 + rank 6: 0.78 + rank 7: 0.83 + rank 8: 0.84 + rank 9: 0.85 + rank 10: 0.85 + rank 11: 0.84 + rank 12: 0.85 + rank 13: 0.73 + rank 14: 0.88 + rank 15: 0.83 + rank 16: 0.85 + rank 17: 0.84 + rank 18: 0.89 + rank 19: 0.88 + rank 20: 0.83 + rank 21: 0.83 + rank 22: 0.81 + rank 23: 0.74 + rank 24: 1.33 + rank 25: 1.37 + rank 26: 1.13 + rank 27: 1.14 + rank 28: 1.32 + rank 29: 1.39 + rank 30: 1.23 + rank 31: 1.39 + rank 32: 0.95 + rank 33: 1.00 + rank 34: 0.96 + rank 35: 0.88 + rank 36: 0.82 + rank 37: 0.92 + rank 38: 0.82 + rank 39: 0.82 + rank 40: 0.85 + rank 41: 0.83 + rank 42: 0.83 + rank 43: 0.75 + rank 44: 0.73 + rank 45: 0.80 + rank 46: 0.81 + rank 47: 0.82 + rank 48: 1.31 + rank 49: 1.22 + rank 50: 1.04 + rank 51: 1.18 + rank 52: 1.26 + rank 53: 1.23 + rank 54: 1.22 + rank 55: 1.29 + rank 56: 0.84 + rank 57: 0.74 + rank 58: 0.75 + rank 59: 0.75 + rank 60: 0.76 + rank 61: 0.74 + rank 62: 0.74 + rank 63: 1.31 + [2026-10-14 04:56:46] iteration 1003/ 2384 | consumed samples: 32096 | elapsed time per iteration (ms): 222193.3 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.81% | FLOPs estimator: workload-aware | logical tokens/s: 75507.1 | valid tokens/s: 75507.1 | physical tokens/s: 76410.6 | loss tokens/s: 54623.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.574635E-01 | lm-global-supervised-tokens: 1.213704E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.325664E-02 | lm-token-buffer-mib: 2.330368E+02 | global_load_balancing_loss: 1.000214E+00 | loss scale: 1.0 | grad norm: 0.224 | params norm: 11186.575 | params rms: 0.060572 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:14:08.979579 | finish at 2026-10-17 18:10:55 +times across ranks (ms): + forward-backward: + rank 0: 221943.52 + rank 1: 221942.53 + rank 2: 221942.23 + rank 3: 221942.30 + rank 4: 221944.88 + rank 5: 221941.88 + rank 6: 221939.75 + rank 7: 221944.41 + rank 8: 221941.16 + rank 9: 221943.22 + rank 10: 221940.91 + rank 11: 221940.19 + rank 12: 221943.59 + rank 13: 221944.17 + rank 14: 221941.75 + rank 15: 221944.62 + rank 16: 221937.97 + rank 17: 221941.56 + rank 18: 221939.66 + rank 19: 221943.09 + rank 20: 221942.69 + rank 21: 221943.77 + rank 22: 221937.16 + rank 23: 221939.44 + rank 24: 221935.86 + rank 25: 221937.52 + rank 26: 221936.14 + rank 27: 221937.25 + rank 28: 221937.70 + rank 29: 221937.38 + rank 30: 221936.70 + rank 31: 221939.09 + rank 32: 221933.20 + rank 33: 221932.70 + rank 34: 221931.77 + rank 35: 221931.88 + rank 36: 221934.34 + rank 37: 221931.48 + rank 38: 221928.97 + rank 39: 221934.16 + rank 40: 221930.39 + rank 41: 221931.23 + rank 42: 221928.09 + rank 43: 221928.06 + rank 44: 221929.48 + rank 45: 221928.86 + rank 46: 221933.11 + rank 47: 221934.41 + rank 48: 221936.06 + rank 49: 221938.45 + rank 50: 221935.59 + rank 51: 221937.78 + rank 52: 221935.97 + rank 53: 221937.78 + rank 54: 221938.06 + rank 55: 221940.34 + rank 56: 221939.31 + rank 57: 221940.56 + rank 58: 221939.31 + rank 59: 221940.00 + rank 60: 221939.53 + rank 61: 221938.70 + rank 62: 221941.05 + rank 63: 221943.33 + forward-compute: + rank 0: 127795.84 + rank 1: 127789.41 + rank 2: 127788.84 + rank 3: 127789.59 + rank 4: 127794.00 + rank 5: 127789.69 + rank 6: 127789.99 + rank 7: 127797.53 + rank 8: 127888.63 + rank 9: 127893.36 + rank 10: 127892.52 + rank 11: 127888.49 + rank 12: 127887.97 + rank 13: 127894.02 + rank 14: 127889.82 + rank 15: 127894.43 + rank 16: 127802.92 + rank 17: 127803.64 + rank 18: 127800.06 + rank 19: 127802.96 + rank 20: 127804.16 + rank 21: 127804.61 + rank 22: 127804.92 + rank 23: 127808.19 + rank 24: 127808.99 + rank 25: 127800.45 + rank 26: 127799.71 + rank 27: 127808.77 + rank 28: 127815.87 + rank 29: 127811.02 + rank 30: 127812.54 + rank 31: 127699.66 + rank 32: 61394.85 + rank 33: 61365.52 + rank 34: 61396.08 + rank 35: 61430.98 + rank 36: 61432.14 + rank 37: 61446.71 + rank 38: 61429.28 + rank 39: 61440.59 + rank 40: 61461.43 + rank 41: 61421.65 + rank 42: 61422.36 + rank 43: 61441.13 + rank 44: 61428.31 + rank 45: 61437.22 + rank 46: 61398.79 + rank 47: 61443.41 + rank 48: 61432.41 + rank 49: 61412.94 + rank 50: 61414.62 + rank 51: 61421.82 + rank 52: 61437.77 + rank 53: 61435.82 + rank 54: 61397.61 + rank 55: 61430.79 + rank 56: 61407.18 + rank 57: 61430.04 + rank 58: 61419.11 + rank 59: 61399.36 + rank 60: 61417.81 + rank 61: 61427.32 + rank 62: 61407.09 + rank 63: 61460.62 + backward-compute: + rank 0: 94090.20 + rank 1: 94095.18 + rank 2: 94092.84 + rank 3: 94093.08 + rank 4: 94091.21 + rank 5: 94095.46 + rank 6: 94090.09 + rank 7: 94088.88 + rank 8: 93998.31 + rank 9: 93991.84 + rank 10: 93991.19 + rank 11: 93997.44 + rank 12: 94001.70 + rank 13: 93996.04 + rank 14: 93996.20 + rank 15: 93995.98 + rank 16: 94085.16 + rank 17: 94086.61 + rank 18: 94085.98 + rank 19: 94086.60 + rank 20: 94087.28 + rank 21: 94086.09 + rank 22: 94083.60 + rank 23: 94083.30 + rank 24: 94032.15 + rank 25: 94039.16 + rank 26: 94036.05 + rank 27: 94027.00 + rank 28: 94024.88 + rank 29: 94030.25 + rank 30: 94026.84 + rank 31: 94137.95 + rank 32: 160493.11 + rank 33: 160521.98 + rank 34: 160491.62 + rank 35: 160454.84 + rank 36: 160458.64 + rank 37: 160440.61 + rank 38: 160456.03 + rank 39: 160449.91 + rank 40: 160434.86 + rank 41: 160476.44 + rank 42: 160471.64 + rank 43: 160452.19 + rank 44: 160467.53 + rank 45: 160458.89 + rank 46: 160499.98 + rank 47: 160456.50 + rank 48: 160469.00 + rank 49: 160490.05 + rank 50: 160484.36 + rank 51: 160485.34 + rank 52: 160463.50 + rank 53: 160468.33 + rank 54: 160503.41 + rank 55: 160473.91 + rank 56: 160499.98 + rank 57: 160475.52 + rank 58: 160486.09 + rank 59: 160505.38 + rank 60: 160487.59 + rank 61: 160475.62 + rank 62: 160500.41 + rank 63: 160448.91 + batch-generator: + rank 0: 45.89 + rank 1: 51.04 + rank 2: 50.33 + rank 3: 49.37 + rank 4: 48.88 + rank 5: 49.28 + rank 6: 48.73 + rank 7: 46.46 + rank 8: 67.28 + rank 9: 68.03 + rank 10: 70.56 + rank 11: 68.54 + rank 12: 67.79 + rank 13: 67.47 + rank 14: 71.07 + rank 15: 66.80 + rank 16: 36.25 + rank 17: 36.47 + rank 18: 36.22 + rank 19: 28.96 + rank 20: 36.32 + rank 21: 37.34 + rank 22: 37.11 + rank 23: 33.83 + rank 24: 49.46 + rank 25: 55.68 + rank 26: 57.97 + rank 27: 59.99 + rank 28: 59.44 + rank 29: 58.42 + rank 30: 52.00 + rank 31: 52.67 + rank 32: 57.35 + rank 33: 63.47 + rank 34: 60.62 + rank 35: 56.06 + rank 36: 51.60 + rank 37: 52.01 + rank 38: 41.44 + rank 39: 54.18 + rank 40: 34.33 + rank 41: 35.16 + rank 42: 33.91 + rank 43: 36.22 + rank 44: 33.01 + rank 45: 33.49 + rank 46: 35.65 + rank 47: 30.35 + rank 48: 38.00 + rank 49: 45.36 + rank 50: 43.66 + rank 51: 40.69 + rank 52: 44.51 + rank 53: 40.96 + rank 54: 37.57 + rank 55: 36.35 + rank 56: 32.54 + rank 57: 38.04 + rank 58: 36.06 + rank 59: 35.49 + rank 60: 35.28 + rank 61: 40.20 + rank 62: 28.55 + rank 63: 31.87 + embedding-grads-all-reduce: + rank 0: 0.03 + rank 1: 0.03 + rank 2: 0.03 + rank 3: 0.03 + rank 4: 0.03 + rank 5: 0.03 + rank 6: 0.03 + rank 7: 0.03 + rank 8: 0.03 + rank 9: 0.03 + rank 10: 0.03 + rank 11: 0.03 + rank 12: 0.03 + rank 13: 0.03 + rank 14: 0.02 + rank 15: 0.03 + rank 16: 0.03 + rank 17: 0.03 + rank 18: 0.03 + rank 19: 0.03 + rank 20: 0.03 + rank 21: 0.02 + rank 22: 0.03 + rank 23: 0.03 + rank 24: 0.04 + rank 25: 0.04 + rank 26: 0.03 + rank 27: 0.04 + rank 28: 0.04 + rank 29: 0.04 + rank 30: 0.04 + rank 31: 0.04 + rank 32: 0.04 + rank 33: 0.03 + rank 34: 0.04 + rank 35: 0.04 + rank 36: 0.03 + rank 37: 0.04 + rank 38: 0.03 + rank 39: 0.04 + rank 40: 0.03 + rank 41: 0.02 + rank 42: 0.03 + rank 43: 0.03 + rank 44: 0.03 + rank 45: 0.03 + rank 46: 0.03 + rank 47: 0.03 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.04 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.03 + rank 57: 0.02 + rank 58: 0.03 + rank 59: 0.02 + rank 60: 0.03 + rank 61: 0.03 + rank 62: 0.03 + rank 63: 0.03 + all-grads-sync: + rank 0: 0.80 + rank 1: 1.06 + rank 2: 0.87 + rank 3: 0.87 + rank 4: 1.24 + rank 5: 0.89 + rank 6: 0.80 + rank 7: 0.84 + rank 8: 0.84 + rank 9: 1.04 + rank 10: 0.88 + rank 11: 0.86 + rank 12: 1.21 + rank 13: 0.93 + rank 14: 1.27 + rank 15: 0.81 + rank 16: 1.27 + rank 17: 0.85 + rank 18: 1.32 + rank 19: 0.94 + rank 20: 0.95 + rank 21: 0.95 + rank 22: 0.90 + rank 23: 1.03 + rank 24: 1.54 + rank 25: 1.27 + rank 26: 1.32 + rank 27: 1.45 + rank 28: 1.72 + rank 29: 1.64 + rank 30: 1.26 + rank 31: 1.44 + rank 32: 1.47 + rank 33: 1.37 + rank 34: 1.60 + rank 35: 1.69 + rank 36: 1.45 + rank 37: 1.29 + rank 38: 1.49 + rank 39: 1.19 + rank 40: 1.02 + rank 41: 1.02 + rank 42: 1.08 + rank 43: 0.80 + rank 44: 0.79 + rank 45: 0.96 + rank 46: 1.19 + rank 47: 1.02 + rank 48: 1.09 + rank 49: 1.31 + rank 50: 1.29 + rank 51: 1.03 + rank 52: 1.11 + rank 53: 1.26 + rank 54: 1.21 + rank 55: 1.38 + rank 56: 0.94 + rank 57: 1.13 + rank 58: 1.10 + rank 59: 1.26 + rank 60: 1.03 + rank 61: 0.93 + rank 62: 0.93 + rank 63: 0.86 + params-all-gather: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.02 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.02 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.02 + rank 20: 0.02 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.03 + rank 24: 0.04 + rank 25: 0.03 + rank 26: 0.03 + rank 27: 0.03 + rank 28: 0.04 + rank 29: 0.03 + rank 30: 0.03 + rank 31: 0.03 + rank 32: 0.02 + rank 33: 0.02 + rank 34: 0.02 + rank 35: 0.03 + rank 36: 0.02 + rank 37: 0.03 + rank 38: 0.03 + rank 39: 0.02 + rank 40: 0.02 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.02 + rank 45: 0.02 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.03 + rank 49: 0.02 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.02 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.02 + optimizer-copy-to-main-grad: + rank 0: 2.25 + rank 1: 1.86 + rank 2: 1.79 + rank 3: 2.17 + rank 4: 1.68 + rank 5: 1.95 + rank 6: 1.94 + rank 7: 1.98 + rank 8: 1.91 + rank 9: 2.16 + rank 10: 1.83 + rank 11: 1.75 + rank 12: 1.87 + rank 13: 1.73 + rank 14: 1.93 + rank 15: 2.09 + rank 16: 2.55 + rank 17: 2.24 + rank 18: 2.26 + rank 19: 2.25 + rank 20: 2.32 + rank 21: 2.41 + rank 22: 2.72 + rank 23: 2.42 + rank 24: 1.97 + rank 25: 2.05 + rank 26: 2.08 + rank 27: 2.19 + rank 28: 2.32 + rank 29: 2.17 + rank 30: 2.18 + rank 31: 2.23 + rank 32: 2.29 + rank 33: 1.95 + rank 34: 2.10 + rank 35: 2.43 + rank 36: 2.15 + rank 37: 2.10 + rank 38: 2.27 + rank 39: 2.14 + rank 40: 1.78 + rank 41: 1.90 + rank 42: 1.83 + rank 43: 1.93 + rank 44: 1.89 + rank 45: 2.09 + rank 46: 1.75 + rank 47: 1.70 + rank 48: 1.92 + rank 49: 1.85 + rank 50: 2.52 + rank 51: 1.76 + rank 52: 1.97 + rank 53: 1.79 + rank 54: 1.81 + rank 55: 1.85 + rank 56: 1.81 + rank 57: 2.38 + rank 58: 1.77 + rank 59: 1.83 + rank 60: 1.76 + rank 61: 1.70 + rank 62: 1.83 + rank 63: 1.75 + optimizer-inner-step: + rank 0: 46.83 + rank 1: 42.69 + rank 2: 42.48 + rank 3: 42.61 + rank 4: 42.31 + rank 5: 42.46 + rank 6: 42.84 + rank 7: 42.76 + rank 8: 42.67 + rank 9: 42.61 + rank 10: 42.51 + rank 11: 42.62 + rank 12: 42.61 + rank 13: 42.91 + rank 14: 42.60 + rank 15: 42.43 + rank 16: 42.89 + rank 17: 42.69 + rank 18: 42.79 + rank 19: 42.63 + rank 20: 42.56 + rank 21: 42.50 + rank 22: 43.18 + rank 23: 42.76 + rank 24: 42.37 + rank 25: 42.67 + rank 26: 42.55 + rank 27: 42.80 + rank 28: 46.36 + rank 29: 42.73 + rank 30: 42.60 + rank 31: 42.77 + rank 32: 42.60 + rank 33: 42.51 + rank 34: 42.62 + rank 35: 49.07 + rank 36: 42.80 + rank 37: 42.64 + rank 38: 42.52 + rank 39: 42.45 + rank 40: 42.55 + rank 41: 42.76 + rank 42: 42.70 + rank 43: 42.88 + rank 44: 42.39 + rank 45: 43.08 + rank 46: 42.69 + rank 47: 42.30 + rank 48: 42.54 + rank 49: 42.50 + rank 50: 54.57 + rank 51: 42.34 + rank 52: 42.72 + rank 53: 42.45 + rank 54: 42.66 + rank 55: 42.61 + rank 56: 42.29 + rank 57: 46.80 + rank 58: 42.58 + rank 59: 42.43 + rank 60: 42.68 + rank 61: 42.42 + rank 62: 42.49 + rank 63: 42.37 + optimizer-copy-main-to-model-params: + rank 0: 4.41 + rank 1: 3.83 + rank 2: 3.75 + rank 3: 4.40 + rank 4: 3.73 + rank 5: 3.98 + rank 6: 4.00 + rank 7: 4.02 + rank 8: 3.91 + rank 9: 4.46 + rank 10: 3.86 + rank 11: 3.80 + rank 12: 3.99 + rank 13: 3.82 + rank 14: 4.04 + rank 15: 4.28 + rank 16: 4.38 + rank 17: 4.02 + rank 18: 4.03 + rank 19: 4.07 + rank 20: 4.11 + rank 21: 4.35 + rank 22: 4.70 + rank 23: 4.20 + rank 24: 3.93 + rank 25: 4.18 + rank 26: 4.01 + rank 27: 4.12 + rank 28: 4.43 + rank 29: 4.47 + rank 30: 4.34 + rank 31: 4.20 + rank 32: 4.22 + rank 33: 3.98 + rank 34: 4.11 + rank 35: 4.60 + rank 36: 4.30 + rank 37: 4.16 + rank 38: 4.50 + rank 39: 3.96 + rank 40: 3.79 + rank 41: 3.91 + rank 42: 3.93 + rank 43: 4.02 + rank 44: 3.96 + rank 45: 4.23 + rank 46: 3.74 + rank 47: 3.72 + rank 48: 4.05 + rank 49: 3.89 + rank 50: 5.12 + rank 51: 3.95 + rank 52: 4.18 + rank 53: 3.88 + rank 54: 3.90 + rank 55: 3.88 + rank 56: 3.75 + rank 57: 4.75 + rank 58: 3.86 + rank 59: 3.75 + rank 60: 3.72 + rank 61: 3.70 + rank 62: 3.93 + rank 63: 3.67 + optimizer: + rank 0: 59.42 + rank 1: 54.39 + rank 2: 54.00 + rank 3: 55.04 + rank 4: 53.85 + rank 5: 54.33 + rank 6: 54.78 + rank 7: 54.78 + rank 8: 54.47 + rank 9: 55.18 + rank 10: 54.22 + rank 11: 54.22 + rank 12: 54.49 + rank 13: 54.56 + rank 14: 54.54 + rank 15: 54.71 + rank 16: 55.62 + rank 17: 54.83 + rank 18: 54.92 + rank 19: 54.87 + rank 20: 54.81 + rank 21: 55.07 + rank 22: 56.22 + rank 23: 55.23 + rank 24: 54.15 + rank 25: 54.78 + rank 26: 54.39 + rank 27: 54.75 + rank 28: 58.79 + rank 29: 55.23 + rank 30: 54.90 + rank 31: 54.85 + rank 32: 54.71 + rank 33: 54.29 + rank 34: 54.53 + rank 35: 61.74 + rank 36: 54.99 + rank 37: 54.68 + rank 38: 55.02 + rank 39: 54.22 + rank 40: 54.18 + rank 41: 54.53 + rank 42: 54.51 + rank 43: 54.80 + rank 44: 54.25 + rank 45: 55.28 + rank 46: 54.22 + rank 47: 53.85 + rank 48: 54.51 + rank 49: 54.19 + rank 50: 67.90 + rank 51: 54.09 + rank 52: 54.82 + rank 53: 54.17 + rank 54: 54.37 + rank 55: 54.31 + rank 56: 53.89 + rank 57: 59.76 + rank 58: 54.29 + rank 59: 53.96 + rank 60: 54.20 + rank 61: 53.92 + rank 62: 54.27 + rank 63: 53.85 + interval-time: + rank 0: 3.13 + rank 1: 3.15 + rank 2: 3.09 + rank 3: 3.07 + rank 4: 3.08 + rank 5: 3.18 + rank 6: 3.07 + rank 7: 3.13 + rank 8: 0.86 + rank 9: 0.90 + rank 10: 0.89 + rank 11: 0.83 + rank 12: 0.87 + rank 13: 0.74 + rank 14: 0.88 + rank 15: 0.84 + rank 16: 0.84 + rank 17: 0.83 + rank 18: 0.85 + rank 19: 0.87 + rank 20: 0.85 + rank 21: 0.84 + rank 22: 0.75 + rank 23: 0.75 + rank 24: 0.99 + rank 25: 0.97 + rank 26: 0.86 + rank 27: 0.96 + rank 28: 0.94 + rank 29: 0.98 + rank 30: 0.87 + rank 31: 0.99 + rank 32: 0.95 + rank 33: 0.99 + rank 34: 0.95 + rank 35: 0.85 + rank 36: 0.81 + rank 37: 0.86 + rank 38: 0.80 + rank 39: 0.81 + rank 40: 0.86 + rank 41: 0.83 + rank 42: 0.83 + rank 43: 0.76 + rank 44: 0.75 + rank 45: 0.82 + rank 46: 0.81 + rank 47: 0.82 + rank 48: 0.91 + rank 49: 0.86 + rank 50: 0.85 + rank 51: 0.82 + rank 52: 0.90 + rank 53: 0.80 + rank 54: 0.81 + rank 55: 0.91 + rank 56: 0.83 + rank 57: 0.74 + rank 58: 0.74 + rank 59: 0.74 + rank 60: 0.78 + rank 61: 0.74 + rank 62: 0.73 + rank 63: 1.49 + [2026-10-14 05:00:26] iteration 1004/ 2384 | consumed samples: 32128 | elapsed time per iteration (ms): 220817.3 | throughput per GPU (TFLOP/s/GPU): 24.1 | MFU 7.72% | FLOPs estimator: workload-aware | logical tokens/s: 75977.8 | valid tokens/s: 75977.8 | physical tokens/s: 76343.0 | loss tokens/s: 60447.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.591077E-01 | lm-global-supervised-tokens: 1.334785E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.851918E-02 | lm-token-buffer-mib: 2.337662E+02 | global_load_balancing_loss: 1.000571E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11186.546 | params rms: 0.060572 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:38:47.886286 | finish at 2026-10-17 17:39:14 +times across ranks (ms): + forward-backward: + rank 0: 220567.80 + rank 1: 220568.41 + rank 2: 220566.23 + rank 3: 220566.34 + rank 4: 220568.53 + rank 5: 220568.05 + rank 6: 220575.36 + rank 7: 220578.28 + rank 8: 220577.94 + rank 9: 220549.28 + rank 10: 220572.66 + rank 11: 220580.08 + rank 12: 220582.06 + rank 13: 220581.05 + rank 14: 220576.75 + rank 15: 220578.48 + rank 16: 220567.42 + rank 17: 220569.34 + rank 18: 220568.94 + rank 19: 220570.19 + rank 20: 220572.11 + rank 21: 220566.20 + rank 22: 220567.55 + rank 23: 220561.33 + rank 24: 220552.27 + rank 25: 220557.25 + rank 26: 220558.61 + rank 27: 220558.81 + rank 28: 220558.91 + rank 29: 220559.75 + rank 30: 220553.72 + rank 31: 220554.41 + rank 32: 220554.42 + rank 33: 220554.98 + rank 34: 220552.23 + rank 35: 220551.02 + rank 36: 220552.80 + rank 37: 220552.70 + rank 38: 220553.42 + rank 39: 220554.25 + rank 40: 220568.81 + rank 41: 220538.56 + rank 42: 220561.05 + rank 43: 220567.64 + rank 44: 220567.28 + rank 45: 220566.56 + rank 46: 220569.69 + rank 47: 220570.16 + rank 48: 220566.81 + rank 49: 220567.86 + rank 50: 220567.20 + rank 51: 220566.56 + rank 52: 220566.55 + rank 53: 220562.20 + rank 54: 220571.53 + rank 55: 220563.17 + rank 56: 220563.89 + rank 57: 220567.84 + rank 58: 220565.95 + rank 59: 220566.78 + rank 60: 220563.34 + rank 61: 220567.09 + rank 62: 220565.81 + rank 63: 220566.70 + forward-compute: + rank 0: 118777.51 + rank 1: 118778.57 + rank 2: 118768.91 + rank 3: 118763.95 + rank 4: 118766.95 + rank 5: 118778.59 + rank 6: 118776.59 + rank 7: 118787.39 + rank 8: 118784.07 + rank 9: 118761.90 + rank 10: 118785.73 + rank 11: 118788.73 + rank 12: 118796.63 + rank 13: 118785.11 + rank 14: 118779.23 + rank 15: 118798.23 + rank 16: 118764.60 + rank 17: 118763.76 + rank 18: 118763.65 + rank 19: 118758.92 + rank 20: 118758.26 + rank 21: 118753.28 + rank 22: 118762.52 + rank 23: 118752.12 + rank 24: 118741.57 + rank 25: 118747.56 + rank 26: 118559.56 + rank 27: 118755.78 + rank 28: 118758.27 + rank 29: 118755.62 + rank 30: 118755.16 + rank 31: 118753.84 + rank 32: 61748.42 + rank 33: 61779.32 + rank 34: 61763.50 + rank 35: 61764.67 + rank 36: 61814.88 + rank 37: 61809.86 + rank 38: 61801.33 + rank 39: 61799.09 + rank 40: 61830.95 + rank 41: 61773.32 + rank 42: 61785.57 + rank 43: 61797.98 + rank 44: 61790.07 + rank 45: 61811.42 + rank 46: 61794.98 + rank 47: 61817.28 + rank 48: 61771.49 + rank 49: 61784.42 + rank 50: 61760.16 + rank 51: 61765.26 + rank 52: 61818.30 + rank 53: 61799.00 + rank 54: 61773.14 + rank 55: 61776.18 + rank 56: 61776.25 + rank 57: 61777.64 + rank 58: 61775.64 + rank 59: 61766.84 + rank 60: 61787.36 + rank 61: 61792.20 + rank 62: 61757.93 + rank 63: 61800.58 + backward-compute: + rank 0: 101715.71 + rank 1: 101715.58 + rank 2: 101723.92 + rank 3: 101726.45 + rank 4: 101725.62 + rank 5: 101715.59 + rank 6: 101718.52 + rank 7: 101708.23 + rank 8: 101745.87 + rank 9: 101736.14 + rank 10: 101733.26 + rank 11: 101742.26 + rank 12: 101734.08 + rank 13: 101746.09 + rank 14: 101745.78 + rank 15: 101728.83 + rank 16: 101754.13 + rank 17: 101755.41 + rank 18: 101755.40 + rank 19: 101760.73 + rank 20: 101766.39 + rank 21: 101763.52 + rank 22: 101760.23 + rank 23: 101762.80 + rank 24: 101755.04 + rank 25: 101748.86 + rank 26: 101935.46 + rank 27: 101740.84 + rank 28: 101733.04 + rank 29: 101743.24 + rank 30: 101739.91 + rank 31: 101741.12 + rank 32: 158769.42 + rank 33: 158735.42 + rank 34: 158750.53 + rank 35: 158749.20 + rank 36: 158700.52 + rank 37: 158705.00 + rank 38: 158713.41 + rank 39: 158717.31 + rank 40: 158703.03 + rank 41: 158731.66 + rank 42: 158742.25 + rank 43: 158735.64 + rank 44: 158743.62 + rank 45: 158722.55 + rank 46: 158739.50 + rank 47: 158718.06 + rank 48: 158761.56 + rank 49: 158749.17 + rank 50: 158770.03 + rank 51: 158771.00 + rank 52: 158716.75 + rank 53: 158727.80 + rank 54: 158761.56 + rank 55: 158753.84 + rank 56: 158754.44 + rank 57: 158754.20 + rank 58: 158755.83 + rank 59: 158765.98 + rank 60: 158742.27 + rank 61: 158739.36 + rank 62: 158775.80 + rank 63: 158732.58 + batch-generator: + rank 0: 34.45 + rank 1: 36.58 + rank 2: 34.09 + rank 3: 33.64 + rank 4: 36.09 + rank 5: 36.37 + rank 6: 35.96 + rank 7: 35.06 + rank 8: 45.80 + rank 9: 44.14 + rank 10: 43.80 + rank 11: 44.32 + rank 12: 46.54 + rank 13: 43.05 + rank 14: 43.93 + rank 15: 44.98 + rank 16: 31.15 + rank 17: 34.82 + rank 18: 33.02 + rank 19: 27.47 + rank 20: 34.36 + rank 21: 34.39 + rank 22: 32.61 + rank 23: 32.07 + rank 24: 39.76 + rank 25: 44.63 + rank 26: 49.28 + rank 27: 45.54 + rank 28: 46.19 + rank 29: 44.47 + rank 30: 45.12 + rank 31: 46.62 + rank 32: 41.48 + rank 33: 50.77 + rank 34: 44.36 + rank 35: 43.02 + rank 36: 36.51 + rank 37: 40.25 + rank 38: 42.80 + rank 39: 40.34 + rank 40: 32.82 + rank 41: 33.43 + rank 42: 34.81 + rank 43: 35.84 + rank 44: 34.28 + rank 45: 35.65 + rank 46: 34.44 + rank 47: 34.39 + rank 48: 38.70 + rank 49: 39.07 + rank 50: 47.47 + rank 51: 38.47 + rank 52: 37.16 + rank 53: 39.46 + rank 54: 36.52 + rank 55: 31.67 + rank 56: 33.61 + rank 57: 36.31 + rank 58: 36.81 + rank 59: 36.24 + rank 60: 37.42 + rank 61: 35.89 + rank 62: 26.51 + rank 63: 30.12 + embedding-grads-all-reduce: + rank 0: 0.03 + rank 1: 0.03 + rank 2: 0.02 + rank 3: 0.03 + rank 4: 0.03 + rank 5: 0.03 + rank 6: 0.03 + rank 7: 0.04 + rank 8: 0.03 + rank 9: 0.03 + rank 10: 0.03 + rank 11: 0.03 + rank 12: 0.03 + rank 13: 0.03 + rank 14: 0.03 + rank 15: 0.03 + rank 16: 0.03 + rank 17: 0.03 + rank 18: 0.03 + rank 19: 0.03 + rank 20: 0.03 + rank 21: 0.03 + rank 22: 0.03 + rank 23: 0.03 + rank 24: 0.04 + rank 25: 0.04 + rank 26: 0.03 + rank 27: 0.03 + rank 28: 0.04 + rank 29: 0.04 + rank 30: 0.04 + rank 31: 0.04 + rank 32: 0.04 + rank 33: 0.03 + rank 34: 0.03 + rank 35: 0.03 + rank 36: 0.03 + rank 37: 0.04 + rank 38: 0.04 + rank 39: 0.04 + rank 40: 0.03 + rank 41: 0.02 + rank 42: 0.03 + rank 43: 0.03 + rank 44: 0.03 + rank 45: 0.03 + rank 46: 0.03 + rank 47: 0.03 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.04 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.03 + rank 57: 0.03 + rank 58: 0.03 + rank 59: 0.02 + rank 60: 0.03 + rank 61: 0.04 + rank 62: 0.03 + rank 63: 0.03 + all-grads-sync: + rank 0: 0.80 + rank 1: 0.91 + rank 2: 1.07 + rank 3: 0.82 + rank 4: 1.09 + rank 5: 0.88 + rank 6: 1.43 + rank 7: 1.39 + rank 8: 0.90 + rank 9: 0.86 + rank 10: 0.89 + rank 11: 0.96 + rank 12: 1.34 + rank 13: 1.07 + rank 14: 1.11 + rank 15: 1.24 + rank 16: 0.92 + rank 17: 1.21 + rank 18: 1.01 + rank 19: 0.86 + rank 20: 1.12 + rank 21: 1.10 + rank 22: 0.85 + rank 23: 0.94 + rank 24: 1.29 + rank 25: 1.29 + rank 26: 1.30 + rank 27: 1.24 + rank 28: 1.49 + rank 29: 1.51 + rank 30: 1.55 + rank 31: 1.62 + rank 32: 1.41 + rank 33: 1.33 + rank 34: 1.93 + rank 35: 1.39 + rank 36: 1.18 + rank 37: 1.32 + rank 38: 1.21 + rank 39: 1.44 + rank 40: 1.12 + rank 41: 0.84 + rank 42: 0.80 + rank 43: 0.77 + rank 44: 0.84 + rank 45: 0.79 + rank 46: 0.99 + rank 47: 0.92 + rank 48: 1.09 + rank 49: 1.01 + rank 50: 1.51 + rank 51: 0.89 + rank 52: 0.99 + rank 53: 1.32 + rank 54: 1.56 + rank 55: 0.96 + rank 56: 1.20 + rank 57: 0.94 + rank 58: 1.12 + rank 59: 0.82 + rank 60: 1.00 + rank 61: 1.35 + rank 62: 1.01 + rank 63: 1.14 + params-all-gather: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.02 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.03 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.02 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.02 + rank 20: 0.02 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.02 + rank 24: 0.04 + rank 25: 0.04 + rank 26: 0.03 + rank 27: 0.04 + rank 28: 0.04 + rank 29: 0.04 + rank 30: 0.03 + rank 31: 0.03 + rank 32: 0.03 + rank 33: 0.03 + rank 34: 0.03 + rank 35: 0.03 + rank 36: 0.03 + rank 37: 0.03 + rank 38: 0.03 + rank 39: 0.03 + rank 40: 0.02 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.02 + rank 45: 0.02 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.02 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.02 + optimizer-copy-to-main-grad: + rank 0: 2.24 + rank 1: 1.86 + rank 2: 1.80 + rank 3: 2.16 + rank 4: 1.67 + rank 5: 1.87 + rank 6: 3.08 + rank 7: 3.14 + rank 8: 1.90 + rank 9: 2.21 + rank 10: 1.82 + rank 11: 1.75 + rank 12: 1.87 + rank 13: 1.69 + rank 14: 1.93 + rank 15: 2.15 + rank 16: 2.51 + rank 17: 2.24 + rank 18: 2.28 + rank 19: 2.31 + rank 20: 2.19 + rank 21: 2.31 + rank 22: 2.59 + rank 23: 2.39 + rank 24: 1.93 + rank 25: 2.16 + rank 26: 2.04 + rank 27: 2.13 + rank 28: 2.40 + rank 29: 2.30 + rank 30: 2.13 + rank 31: 2.22 + rank 32: 2.10 + rank 33: 2.05 + rank 34: 2.11 + rank 35: 2.39 + rank 36: 2.22 + rank 37: 2.24 + rank 38: 2.41 + rank 39: 2.05 + rank 40: 1.76 + rank 41: 1.93 + rank 42: 1.91 + rank 43: 1.91 + rank 44: 1.92 + rank 45: 2.09 + rank 46: 1.78 + rank 47: 1.76 + rank 48: 1.92 + rank 49: 1.79 + rank 50: 2.50 + rank 51: 1.73 + rank 52: 1.92 + rank 53: 1.77 + rank 54: 1.76 + rank 55: 1.80 + rank 56: 1.79 + rank 57: 2.49 + rank 58: 1.78 + rank 59: 1.75 + rank 60: 1.79 + rank 61: 1.70 + rank 62: 1.84 + rank 63: 1.73 + optimizer-inner-step: + rank 0: 43.07 + rank 1: 40.10 + rank 2: 39.83 + rank 3: 40.07 + rank 4: 39.78 + rank 5: 40.04 + rank 6: 43.56 + rank 7: 40.51 + rank 8: 42.44 + rank 9: 42.39 + rank 10: 42.23 + rank 11: 42.33 + rank 12: 42.38 + rank 13: 42.57 + rank 14: 42.40 + rank 15: 42.23 + rank 16: 42.77 + rank 17: 42.45 + rank 18: 42.53 + rank 19: 42.34 + rank 20: 42.35 + rank 21: 42.25 + rank 22: 42.89 + rank 23: 42.51 + rank 24: 42.06 + rank 25: 42.22 + rank 26: 42.24 + rank 27: 45.38 + rank 28: 46.82 + rank 29: 42.06 + rank 30: 42.10 + rank 31: 43.57 + rank 32: 42.37 + rank 33: 42.26 + rank 34: 42.35 + rank 35: 48.94 + rank 36: 42.51 + rank 37: 42.35 + rank 38: 42.20 + rank 39: 42.19 + rank 40: 42.36 + rank 41: 42.49 + rank 42: 42.51 + rank 43: 42.61 + rank 44: 42.16 + rank 45: 42.80 + rank 46: 42.44 + rank 47: 42.07 + rank 48: 42.39 + rank 49: 42.27 + rank 50: 54.37 + rank 51: 42.12 + rank 52: 42.44 + rank 53: 42.26 + rank 54: 42.49 + rank 55: 42.38 + rank 56: 42.03 + rank 57: 46.97 + rank 58: 42.30 + rank 59: 42.15 + rank 60: 42.52 + rank 61: 42.14 + rank 62: 42.27 + rank 63: 42.18 + optimizer-copy-main-to-model-params: + rank 0: 4.42 + rank 1: 3.87 + rank 2: 3.73 + rank 3: 4.41 + rank 4: 3.67 + rank 5: 3.96 + rank 6: 3.95 + rank 7: 4.01 + rank 8: 3.95 + rank 9: 4.58 + rank 10: 3.88 + rank 11: 3.81 + rank 12: 3.96 + rank 13: 3.78 + rank 14: 4.04 + rank 15: 4.24 + rank 16: 4.43 + rank 17: 4.01 + rank 18: 4.08 + rank 19: 4.05 + rank 20: 3.99 + rank 21: 4.25 + rank 22: 4.61 + rank 23: 4.19 + rank 24: 3.92 + rank 25: 4.35 + rank 26: 4.13 + rank 27: 4.26 + rank 28: 4.66 + rank 29: 4.51 + rank 30: 4.47 + rank 31: 4.10 + rank 32: 4.09 + rank 33: 3.93 + rank 34: 4.06 + rank 35: 4.56 + rank 36: 4.20 + rank 37: 4.05 + rank 38: 4.46 + rank 39: 3.89 + rank 40: 3.87 + rank 41: 4.05 + rank 42: 4.08 + rank 43: 4.11 + rank 44: 4.00 + rank 45: 4.27 + rank 46: 3.77 + rank 47: 3.80 + rank 48: 4.07 + rank 49: 3.88 + rank 50: 4.97 + rank 51: 3.97 + rank 52: 4.17 + rank 53: 3.87 + rank 54: 3.92 + rank 55: 3.88 + rank 56: 3.78 + rank 57: 4.75 + rank 58: 3.97 + rank 59: 3.79 + rank 60: 3.77 + rank 61: 3.69 + rank 62: 3.92 + rank 63: 3.68 + optimizer: + rank 0: 57.73 + rank 1: 53.94 + rank 2: 53.47 + rank 3: 54.66 + rank 4: 53.39 + rank 5: 53.98 + rank 6: 58.02 + rank 7: 55.10 + rank 8: 54.03 + rank 9: 54.88 + rank 10: 53.77 + rank 11: 53.75 + rank 12: 54.02 + rank 13: 53.96 + rank 14: 54.12 + rank 15: 54.24 + rank 16: 55.29 + rank 17: 54.38 + rank 18: 54.50 + rank 19: 54.35 + rank 20: 54.27 + rank 21: 54.52 + rank 22: 55.63 + rank 23: 54.78 + rank 24: 53.65 + rank 25: 54.47 + rank 26: 54.11 + rank 27: 57.36 + rank 28: 59.48 + rank 29: 54.55 + rank 30: 54.55 + rank 31: 55.37 + rank 32: 54.14 + rank 33: 53.78 + rank 34: 54.00 + rank 35: 61.34 + rank 36: 54.41 + rank 37: 54.06 + rank 38: 54.45 + rank 39: 53.67 + rank 40: 53.84 + rank 41: 54.19 + rank 42: 54.26 + rank 43: 54.39 + rank 44: 53.83 + rank 45: 54.85 + rank 46: 53.78 + rank 47: 53.45 + rank 48: 54.13 + rank 49: 53.74 + rank 50: 67.35 + rank 51: 53.69 + rank 52: 54.32 + rank 53: 53.74 + rank 54: 53.98 + rank 55: 53.84 + rank 56: 53.45 + rank 57: 59.73 + rank 58: 53.93 + rank 59: 53.53 + rank 60: 53.87 + rank 61: 53.41 + rank 62: 53.82 + rank 63: 53.47 + interval-time: + rank 0: 0.97 + rank 1: 1.00 + rank 2: 0.94 + rank 3: 0.92 + rank 4: 1.07 + rank 5: 1.05 + rank 6: 0.96 + rank 7: 1.01 + rank 8: 0.85 + rank 9: 0.85 + rank 10: 0.86 + rank 11: 0.84 + rank 12: 0.85 + rank 13: 0.74 + rank 14: 0.87 + rank 15: 0.86 + rank 16: 0.85 + rank 17: 0.84 + rank 18: 0.87 + rank 19: 0.88 + rank 20: 0.82 + rank 21: 0.82 + rank 22: 0.75 + rank 23: 0.73 + rank 24: 0.97 + rank 25: 0.95 + rank 26: 0.85 + rank 27: 0.96 + rank 28: 1.03 + rank 29: 0.98 + rank 30: 0.86 + rank 31: 0.98 + rank 32: 1.18 + rank 33: 1.32 + rank 34: 1.27 + rank 35: 1.09 + rank 36: 1.11 + rank 37: 1.19 + rank 38: 1.14 + rank 39: 1.18 + rank 40: 0.87 + rank 41: 0.84 + rank 42: 0.85 + rank 43: 0.76 + rank 44: 0.73 + rank 45: 0.82 + rank 46: 0.83 + rank 47: 0.84 + rank 48: 0.91 + rank 49: 0.86 + rank 50: 0.86 + rank 51: 0.81 + rank 52: 0.90 + rank 53: 0.81 + rank 54: 0.82 + rank 55: 0.90 + rank 56: 0.86 + rank 57: 0.76 + rank 58: 0.77 + rank 59: 0.79 + rank 60: 0.78 + rank 61: 0.76 + rank 62: 0.77 + rank 63: 1.34 + [2026-10-14 05:04:13] iteration 1005/ 2384 | consumed samples: 32160 | elapsed time per iteration (ms): 226128.2 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.27% | FLOPs estimator: workload-aware | logical tokens/s: 74193.2 | valid tokens/s: 74193.2 | physical tokens/s: 75193.9 | loss tokens/s: 55426.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.115333E-01 | lm-global-supervised-tokens: 1.253340E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.337362E-02 | lm-token-buffer-mib: 2.335799E+02 | global_load_balancing_loss: 9.999340E-01 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11186.516 | params rms: 0.060571 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:37:10.721461 | finish at 2026-10-17 19:41:23 +times across ranks (ms): + forward-backward: + rank 0: 225883.95 + rank 1: 225886.86 + rank 2: 225885.53 + rank 3: 225882.33 + rank 4: 225888.53 + rank 5: 225886.25 + rank 6: 225879.95 + rank 7: 225883.66 + rank 8: 225893.84 + rank 9: 225894.86 + rank 10: 225895.34 + rank 11: 225896.70 + rank 12: 225897.19 + rank 13: 225899.08 + rank 14: 225893.64 + rank 15: 225895.17 + rank 16: 225883.36 + rank 17: 225887.86 + rank 18: 225888.12 + rank 19: 225875.28 + rank 20: 225899.23 + rank 21: 225900.53 + rank 22: 225884.59 + rank 23: 225885.08 + rank 24: 225877.72 + rank 25: 225882.38 + rank 26: 225884.06 + rank 27: 225883.27 + rank 28: 225882.62 + rank 29: 225881.58 + rank 30: 225878.55 + rank 31: 225880.17 + rank 32: 225870.66 + rank 33: 225874.98 + rank 34: 225871.61 + rank 35: 225866.97 + rank 36: 225875.42 + rank 37: 225870.81 + rank 38: 225866.78 + rank 39: 225872.75 + rank 40: 225887.91 + rank 41: 225887.23 + rank 42: 225886.66 + rank 43: 225887.03 + rank 44: 225888.05 + rank 45: 225887.02 + rank 46: 225887.94 + rank 47: 225888.61 + rank 48: 225882.98 + rank 49: 225887.45 + rank 50: 225884.05 + rank 51: 225873.27 + rank 52: 225883.33 + rank 53: 225885.08 + rank 54: 225885.31 + rank 55: 225885.95 + rank 56: 225886.55 + rank 57: 225888.00 + rank 58: 225889.70 + rank 59: 225888.25 + rank 60: 225885.78 + rank 61: 225885.44 + rank 62: 225887.14 + rank 63: 225889.31 + forward-compute: + rank 0: 130875.39 + rank 1: 130872.51 + rank 2: 130868.30 + rank 3: 130865.03 + rank 4: 130870.98 + rank 5: 130867.87 + rank 6: 130869.79 + rank 7: 130874.18 + rank 8: 130949.15 + rank 9: 130951.85 + rank 10: 130957.10 + rank 11: 130950.97 + rank 12: 130955.42 + rank 13: 130965.43 + rank 14: 130941.19 + rank 15: 130953.20 + rank 16: 130874.67 + rank 17: 130874.75 + rank 18: 130871.12 + rank 19: 130864.52 + rank 20: 130875.94 + rank 21: 130879.65 + rank 22: 130882.80 + rank 23: 130882.50 + rank 24: 130882.10 + rank 25: 130882.13 + rank 26: 130892.41 + rank 27: 130890.42 + rank 28: 130887.08 + rank 29: 130883.37 + rank 30: 130889.06 + rank 31: 130886.07 + rank 32: 61572.04 + rank 33: 61591.63 + rank 34: 61599.48 + rank 35: 61615.38 + rank 36: 61635.86 + rank 37: 61634.29 + rank 38: 61612.77 + rank 39: 61619.16 + rank 40: 61670.98 + rank 41: 61643.32 + rank 42: 61648.50 + rank 43: 61660.82 + rank 44: 61645.52 + rank 45: 61658.84 + rank 46: 61620.07 + rank 47: 61659.91 + rank 48: 61627.00 + rank 49: 61622.99 + rank 50: 61628.82 + rank 51: 61616.28 + rank 52: 61653.68 + rank 53: 61639.59 + rank 54: 61606.59 + rank 55: 61613.41 + rank 56: 61629.70 + rank 57: 61608.16 + rank 58: 61600.32 + rank 59: 61616.07 + rank 60: 61610.99 + rank 61: 61643.02 + rank 62: 61586.53 + rank 63: 61644.88 + backward-compute: + rank 0: 94957.87 + rank 1: 94963.87 + rank 2: 94962.67 + rank 3: 94963.44 + rank 4: 94961.21 + rank 5: 94965.22 + rank 6: 94959.77 + rank 7: 94961.09 + rank 8: 94896.29 + rank 9: 94889.04 + rank 10: 94885.66 + rank 11: 94894.52 + rank 12: 94887.79 + rank 13: 94880.45 + rank 14: 94897.06 + rank 15: 94889.59 + rank 16: 94959.80 + rank 17: 94961.38 + rank 18: 94963.16 + rank 19: 94958.77 + rank 20: 94960.98 + rank 21: 94957.68 + rank 22: 94955.49 + rank 23: 94952.95 + rank 24: 94904.02 + rank 25: 94900.63 + rank 26: 94888.80 + rank 27: 94891.73 + rank 28: 94896.04 + rank 29: 94902.38 + rank 30: 94894.53 + rank 31: 94893.57 + rank 32: 164260.30 + rank 33: 164242.69 + rank 34: 164234.39 + rank 35: 164213.73 + rank 36: 164201.38 + rank 37: 164198.91 + rank 38: 164213.38 + rank 39: 164215.75 + rank 40: 164178.94 + rank 41: 164209.27 + rank 42: 164205.20 + rank 43: 164193.00 + rank 44: 164207.41 + rank 45: 164195.05 + rank 46: 164232.64 + rank 47: 164193.83 + rank 48: 164219.94 + rank 49: 164228.98 + rank 50: 164218.66 + rank 51: 164223.23 + rank 52: 164197.80 + rank 53: 164211.16 + rank 54: 164241.92 + rank 55: 164240.56 + rank 56: 164224.67 + rank 57: 164245.11 + rank 58: 164254.53 + rank 59: 164238.17 + rank 60: 164240.00 + rank 61: 164208.56 + rank 62: 164267.89 + rank 63: 164211.09 + batch-generator: + rank 0: 32.72 + rank 1: 35.79 + rank 2: 36.22 + rank 3: 34.93 + rank 4: 36.61 + rank 5: 37.15 + rank 6: 37.49 + rank 7: 35.66 + rank 8: 57.70 + rank 9: 60.26 + rank 10: 63.49 + rank 11: 61.22 + rank 12: 60.36 + rank 13: 61.81 + rank 14: 57.20 + rank 15: 60.19 + rank 16: 34.63 + rank 17: 37.45 + rank 18: 35.39 + rank 19: 28.03 + rank 20: 35.88 + rank 21: 35.45 + rank 22: 33.76 + rank 23: 34.27 + rank 24: 41.07 + rank 25: 46.71 + rank 26: 52.83 + rank 27: 54.62 + rank 28: 49.28 + rank 29: 49.28 + rank 30: 47.67 + rank 31: 47.45 + rank 32: 44.42 + rank 33: 50.35 + rank 34: 41.37 + rank 35: 40.49 + rank 36: 37.81 + rank 37: 39.84 + rank 38: 41.15 + rank 39: 40.27 + rank 40: 33.85 + rank 41: 35.03 + rank 42: 34.59 + rank 43: 35.20 + rank 44: 32.75 + rank 45: 32.00 + rank 46: 34.03 + rank 47: 31.14 + rank 48: 38.86 + rank 49: 37.65 + rank 50: 45.51 + rank 51: 43.87 + rank 52: 39.64 + rank 53: 39.58 + rank 54: 35.13 + rank 55: 30.90 + rank 56: 31.89 + rank 57: 35.77 + rank 58: 38.75 + rank 59: 35.79 + rank 60: 35.81 + rank 61: 35.82 + rank 62: 27.11 + rank 63: 29.02 + embedding-grads-all-reduce: + rank 0: 0.03 + rank 1: 0.03 + rank 2: 0.03 + rank 3: 0.03 + rank 4: 0.03 + rank 5: 0.03 + rank 6: 0.02 + rank 7: 0.03 + rank 8: 0.03 + rank 9: 0.03 + rank 10: 0.03 + rank 11: 0.03 + rank 12: 0.03 + rank 13: 0.03 + rank 14: 0.02 + rank 15: 0.03 + rank 16: 0.03 + rank 17: 0.03 + rank 18: 0.04 + rank 19: 0.03 + rank 20: 0.04 + rank 21: 0.03 + rank 22: 0.03 + rank 23: 0.03 + rank 24: 0.04 + rank 25: 0.04 + rank 26: 0.04 + rank 27: 0.04 + rank 28: 0.04 + rank 29: 0.04 + rank 30: 0.04 + rank 31: 0.04 + rank 32: 0.03 + rank 33: 0.03 + rank 34: 0.04 + rank 35: 0.04 + rank 36: 0.03 + rank 37: 0.04 + rank 38: 0.03 + rank 39: 0.04 + rank 40: 0.03 + rank 41: 0.03 + rank 42: 0.03 + rank 43: 0.03 + rank 44: 0.03 + rank 45: 0.03 + rank 46: 0.03 + rank 47: 0.03 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.03 + rank 57: 0.03 + rank 58: 0.03 + rank 59: 0.03 + rank 60: 0.03 + rank 61: 0.03 + rank 62: 0.03 + rank 63: 0.03 + all-grads-sync: + rank 0: 1.11 + rank 1: 1.04 + rank 2: 0.81 + rank 3: 0.82 + rank 4: 0.88 + rank 5: 1.19 + rank 6: 0.79 + rank 7: 1.07 + rank 8: 0.87 + rank 9: 1.02 + rank 10: 0.96 + rank 11: 1.26 + rank 12: 0.87 + rank 13: 1.01 + rank 14: 0.81 + rank 15: 1.04 + rank 16: 0.93 + rank 17: 0.92 + rank 18: 1.22 + rank 19: 0.94 + rank 20: 1.53 + rank 21: 1.61 + rank 22: 1.29 + rank 23: 0.91 + rank 24: 1.54 + rank 25: 1.33 + rank 26: 1.37 + rank 27: 1.28 + rank 28: 1.31 + rank 29: 1.46 + rank 30: 1.21 + rank 31: 1.67 + rank 32: 1.22 + rank 33: 1.81 + rank 34: 1.57 + rank 35: 1.29 + rank 36: 1.37 + rank 37: 1.27 + rank 38: 1.02 + rank 39: 1.29 + rank 40: 1.24 + rank 41: 1.05 + rank 42: 0.84 + rank 43: 0.88 + rank 44: 0.81 + rank 45: 0.88 + rank 46: 1.13 + rank 47: 1.09 + rank 48: 1.33 + rank 49: 1.37 + rank 50: 1.38 + rank 51: 0.98 + rank 52: 0.95 + rank 53: 0.92 + rank 54: 1.02 + rank 55: 0.86 + rank 56: 1.09 + rank 57: 0.93 + rank 58: 0.83 + rank 59: 1.14 + rank 60: 0.87 + rank 61: 0.89 + rank 62: 1.04 + rank 63: 0.84 + params-all-gather: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.03 + rank 5: 0.02 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.03 + rank 16: 0.02 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.02 + rank 20: 0.02 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.02 + rank 24: 0.04 + rank 25: 0.04 + rank 26: 0.04 + rank 27: 0.04 + rank 28: 0.03 + rank 29: 0.04 + rank 30: 0.04 + rank 31: 0.03 + rank 32: 0.02 + rank 33: 0.03 + rank 34: 0.02 + rank 35: 0.02 + rank 36: 0.03 + rank 37: 0.02 + rank 38: 0.03 + rank 39: 0.02 + rank 40: 0.02 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.02 + rank 45: 0.02 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.03 + rank 49: 0.02 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.02 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.02 + optimizer-copy-to-main-grad: + rank 0: 2.21 + rank 1: 1.83 + rank 2: 1.86 + rank 3: 2.35 + rank 4: 1.71 + rank 5: 1.97 + rank 6: 1.94 + rank 7: 2.07 + rank 8: 1.85 + rank 9: 2.23 + rank 10: 1.81 + rank 11: 1.75 + rank 12: 1.90 + rank 13: 1.75 + rank 14: 1.91 + rank 15: 2.26 + rank 16: 2.61 + rank 17: 2.25 + rank 18: 2.32 + rank 19: 2.26 + rank 20: 3.65 + rank 21: 3.75 + rank 22: 2.70 + rank 23: 2.44 + rank 24: 2.00 + rank 25: 2.14 + rank 26: 2.20 + rank 27: 2.26 + rank 28: 2.32 + rank 29: 2.16 + rank 30: 2.36 + rank 31: 2.20 + rank 32: 2.08 + rank 33: 1.93 + rank 34: 1.92 + rank 35: 2.23 + rank 36: 2.17 + rank 37: 2.08 + rank 38: 2.41 + rank 39: 2.08 + rank 40: 1.82 + rank 41: 1.98 + rank 42: 1.84 + rank 43: 1.91 + rank 44: 1.99 + rank 45: 2.12 + rank 46: 1.74 + rank 47: 1.72 + rank 48: 1.91 + rank 49: 1.85 + rank 50: 2.52 + rank 51: 1.84 + rank 52: 1.96 + rank 53: 1.75 + rank 54: 1.73 + rank 55: 1.81 + rank 56: 1.83 + rank 57: 2.44 + rank 58: 1.82 + rank 59: 1.80 + rank 60: 1.79 + rank 61: 1.68 + rank 62: 1.79 + rank 63: 1.72 + optimizer-inner-step: + rank 0: 42.66 + rank 1: 42.70 + rank 2: 42.33 + rank 3: 42.49 + rank 4: 42.32 + rank 5: 42.52 + rank 6: 42.86 + rank 7: 42.89 + rank 8: 42.66 + rank 9: 42.63 + rank 10: 42.47 + rank 11: 42.59 + rank 12: 42.60 + rank 13: 42.83 + rank 14: 42.65 + rank 15: 42.41 + rank 16: 43.00 + rank 17: 42.69 + rank 18: 42.78 + rank 19: 42.73 + rank 20: 41.75 + rank 21: 41.79 + rank 22: 43.14 + rank 23: 42.77 + rank 24: 38.82 + rank 25: 42.87 + rank 26: 40.25 + rank 27: 44.30 + rank 28: 48.09 + rank 29: 43.30 + rank 30: 42.25 + rank 31: 43.92 + rank 32: 42.70 + rank 33: 42.60 + rank 34: 42.70 + rank 35: 49.28 + rank 36: 42.77 + rank 37: 42.73 + rank 38: 42.54 + rank 39: 42.55 + rank 40: 42.54 + rank 41: 42.71 + rank 42: 42.74 + rank 43: 42.83 + rank 44: 42.42 + rank 45: 42.96 + rank 46: 42.63 + rank 47: 42.39 + rank 48: 42.69 + rank 49: 42.53 + rank 50: 54.44 + rank 51: 42.36 + rank 52: 42.75 + rank 53: 42.56 + rank 54: 42.75 + rank 55: 42.69 + rank 56: 42.32 + rank 57: 46.49 + rank 58: 42.59 + rank 59: 42.46 + rank 60: 42.81 + rank 61: 42.46 + rank 62: 42.50 + rank 63: 42.49 + optimizer-copy-main-to-model-params: + rank 0: 4.39 + rank 1: 3.84 + rank 2: 3.72 + rank 3: 4.38 + rank 4: 3.70 + rank 5: 3.96 + rank 6: 4.03 + rank 7: 3.96 + rank 8: 3.82 + rank 9: 4.47 + rank 10: 3.85 + rank 11: 3.78 + rank 12: 3.95 + rank 13: 3.81 + rank 14: 3.99 + rank 15: 4.27 + rank 16: 4.50 + rank 17: 4.01 + rank 18: 4.12 + rank 19: 4.06 + rank 20: 4.03 + rank 21: 4.21 + rank 22: 4.61 + rank 23: 4.20 + rank 24: 3.91 + rank 25: 4.21 + rank 26: 4.03 + rank 27: 4.07 + rank 28: 4.43 + rank 29: 4.41 + rank 30: 4.39 + rank 31: 4.05 + rank 32: 4.25 + rank 33: 4.09 + rank 34: 4.05 + rank 35: 4.69 + rank 36: 4.28 + rank 37: 4.31 + rank 38: 4.46 + rank 39: 4.14 + rank 40: 3.79 + rank 41: 3.99 + rank 42: 4.01 + rank 43: 4.01 + rank 44: 4.01 + rank 45: 4.23 + rank 46: 3.82 + rank 47: 3.73 + rank 48: 4.05 + rank 49: 3.89 + rank 50: 4.98 + rank 51: 3.99 + rank 52: 4.16 + rank 53: 3.87 + rank 54: 3.88 + rank 55: 3.90 + rank 56: 3.81 + rank 57: 4.68 + rank 58: 3.90 + rank 59: 3.73 + rank 60: 3.79 + rank 61: 3.81 + rank 62: 3.96 + rank 63: 3.76 + optimizer: + rank 0: 58.10 + rank 1: 57.28 + rank 2: 56.75 + rank 3: 57.81 + rank 4: 56.68 + rank 5: 57.19 + rank 6: 57.66 + rank 7: 57.62 + rank 8: 57.15 + rank 9: 58.05 + rank 10: 56.99 + rank 11: 57.02 + rank 12: 57.30 + rank 13: 57.34 + rank 14: 57.31 + rank 15: 57.57 + rank 16: 58.66 + rank 17: 57.64 + rank 18: 57.86 + rank 19: 57.68 + rank 20: 57.55 + rank 21: 57.79 + rank 22: 58.92 + rank 23: 58.09 + rank 24: 56.97 + rank 25: 61.46 + rank 26: 58.51 + rank 27: 62.62 + rank 28: 66.95 + rank 29: 62.15 + rank 30: 61.02 + rank 31: 62.21 + rank 32: 57.61 + rank 33: 57.24 + rank 34: 57.34 + rank 35: 64.81 + rank 36: 57.77 + rank 37: 57.68 + rank 38: 57.80 + rank 39: 57.27 + rank 40: 57.01 + rank 41: 57.40 + rank 42: 57.46 + rank 43: 57.55 + rank 44: 57.11 + rank 45: 58.05 + rank 46: 57.09 + rank 47: 56.74 + rank 48: 57.40 + rank 49: 57.05 + rank 50: 70.52 + rank 51: 56.96 + rank 52: 57.64 + rank 53: 57.03 + rank 54: 57.24 + rank 55: 57.17 + rank 56: 56.74 + rank 57: 62.29 + rank 58: 57.15 + rank 59: 56.76 + rank 60: 57.18 + rank 61: 56.86 + rank 62: 57.15 + rank 63: 56.82 + interval-time: + rank 0: 0.81 + rank 1: 0.83 + rank 2: 0.76 + rank 3: 0.74 + rank 4: 0.73 + rank 5: 0.87 + rank 6: 0.77 + rank 7: 0.81 + rank 8: 0.82 + rank 9: 0.85 + rank 10: 0.84 + rank 11: 0.83 + rank 12: 0.84 + rank 13: 0.73 + rank 14: 0.83 + rank 15: 0.85 + rank 16: 0.89 + rank 17: 0.84 + rank 18: 0.86 + rank 19: 0.88 + rank 20: 0.84 + rank 21: 0.82 + rank 22: 0.74 + rank 23: 0.74 + rank 24: 1.37 + rank 25: 1.32 + rank 26: 1.17 + rank 27: 1.20 + rank 28: 1.28 + rank 29: 1.31 + rank 30: 1.26 + rank 31: 1.39 + rank 32: 0.94 + rank 33: 1.00 + rank 34: 0.96 + rank 35: 0.86 + rank 36: 0.80 + rank 37: 0.83 + rank 38: 0.82 + rank 39: 0.81 + rank 40: 0.85 + rank 41: 0.83 + rank 42: 0.83 + rank 43: 0.76 + rank 44: 0.73 + rank 45: 0.80 + rank 46: 0.83 + rank 47: 0.81 + rank 48: 0.92 + rank 49: 0.85 + rank 50: 0.85 + rank 51: 0.82 + rank 52: 0.91 + rank 53: 0.80 + rank 54: 0.82 + rank 55: 0.92 + rank 56: 0.82 + rank 57: 0.72 + rank 58: 0.72 + rank 59: 0.73 + rank 60: 0.74 + rank 61: 0.74 + rank 62: 0.73 + rank 63: 1.30 + [2026-10-14 05:07:57] iteration 1006/ 2384 | consumed samples: 32192 | elapsed time per iteration (ms): 224351.0 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.54% | FLOPs estimator: workload-aware | logical tokens/s: 74781.1 | valid tokens/s: 74781.1 | physical tokens/s: 75939.6 | loss tokens/s: 58250.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.972449E-01 | lm-global-supervised-tokens: 1.306859E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.692392E-02 | lm-token-buffer-mib: 2.356137E+02 | global_load_balancing_loss: 1.000628E+00 | loss scale: 1.0 | grad norm: 0.178 | params norm: 11186.487 | params rms: 0.060571 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:52:35.636701 | finish at 2026-10-17 19:00:33 + [2026-10-14 05:11:41] iteration 1007/ 2384 | consumed samples: 32224 | elapsed time per iteration (ms): 223993.3 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.58% | FLOPs estimator: workload-aware | logical tokens/s: 74900.4 | valid tokens/s: 74900.4 | physical tokens/s: 75689.1 | loss tokens/s: 56238.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.757658E-01 | lm-global-supervised-tokens: 1.259708E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.557074E-02 | lm-token-buffer-mib: 2.341348E+02 | global_load_balancing_loss: 1.000842E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11186.459 | params rms: 0.060571 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:40:38.707401 | finish at 2026-10-17 18:52:20 + [2026-10-14 05:15:22] iteration 1008/ 2384 | consumed samples: 32256 | elapsed time per iteration (ms): 221207.4 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.84% | FLOPs estimator: workload-aware | logical tokens/s: 75843.7 | valid tokens/s: 75843.7 | physical tokens/s: 76372.7 | loss tokens/s: 54627.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.072842E-01 | lm-global-supervised-tokens: 1.208398E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.464093E-02 | lm-token-buffer-mib: 2.339794E+02 | global_load_balancing_loss: 1.000536E+00 | loss scale: 1.0 | grad norm: 0.135 | params norm: 11186.432 | params rms: 0.060571 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:33:01.348724 | finish at 2026-10-17 17:48:24 + [2026-10-14 05:19:04] iteration 1009/ 2384 | consumed samples: 32288 | elapsed time per iteration (ms): 221542.3 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.03% | FLOPs estimator: workload-aware | logical tokens/s: 75729.1 | valid tokens/s: 75729.1 | physical tokens/s: 76669.5 | loss tokens/s: 54383.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.980495E-01 | lm-global-supervised-tokens: 1.204817E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.841899E-02 | lm-token-buffer-mib: 2.355093E+02 | global_load_balancing_loss: 1.000445E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11186.403 | params rms: 0.060571 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:37:00.655596 | finish at 2026-10-17 17:56:04 + [2026-10-14 05:23:07] iteration 1010/ 2384 | consumed samples: 32320 | elapsed time per iteration (ms): 242793.7 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.24% | FLOPs estimator: workload-aware | logical tokens/s: 69100.6 | valid tokens/s: 69100.6 | physical tokens/s: 69687.2 | loss tokens/s: 48995.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.907034E-01 | lm-global-supervised-tokens: 1.189568E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.248956E-02 | lm-token-buffer-mib: 2.343055E+02 | global_load_balancing_loss: 1.000151E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11186.372 | params rms: 0.060571 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:39:58.506755 | finish at 2026-10-18 02:03:05 + [2026-10-14 05:26:50] iteration 1011/ 2384 | consumed samples: 32352 | elapsed time per iteration (ms): 223350.3 | throughput per GPU (TFLOP/s/GPU): 18.9 | MFU 6.07% | FLOPs estimator: workload-aware | logical tokens/s: 75116.1 | valid tokens/s: 75116.1 | physical tokens/s: 76447.2 | loss tokens/s: 56026.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.668032E-01 | lm-global-supervised-tokens: 1.251353E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.451439E-02 | lm-token-buffer-mib: 2.334359E+02 | global_load_balancing_loss: 1.000672E+00 | loss scale: 1.0 | grad norm: 0.111 | params norm: 11186.340 | params rms: 0.060570 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:10:59.992445 | finish at 2026-10-17 18:37:50 + [2026-10-14 05:30:33] iteration 1012/ 2384 | consumed samples: 32384 | elapsed time per iteration (ms): 223130.4 | throughput per GPU (TFLOP/s/GPU): 28.1 | MFU 9.00% | FLOPs estimator: workload-aware | logical tokens/s: 75189.9 | valid tokens/s: 75189.9 | physical tokens/s: 75534.6 | loss tokens/s: 58517.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.767725E-01 | lm-global-supervised-tokens: 1.305692E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.547235E-02 | lm-token-buffer-mib: 2.337834E+02 | global_load_balancing_loss: 1.000759E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11186.310 | params rms: 0.060570 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:02:14.975779 | finish at 2026-10-17 18:32:48 + [2026-10-14 05:34:20] iteration 1013/ 2384 | consumed samples: 32416 | elapsed time per iteration (ms): 226987.0 | throughput per GPU (TFLOP/s/GPU): 27.2 | MFU 8.72% | FLOPs estimator: workload-aware | logical tokens/s: 73912.5 | valid tokens/s: 73912.5 | physical tokens/s: 74480.0 | loss tokens/s: 56287.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.836886E-01 | lm-global-supervised-tokens: 1.277663E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.288650E-02 | lm-token-buffer-mib: 2.349276E+02 | global_load_balancing_loss: 1.001836E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11186.279 | params rms: 0.060570 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:26:39.181561 | finish at 2026-10-17 20:00:59 + [2026-10-14 05:38:05] iteration 1014/ 2384 | consumed samples: 32448 | elapsed time per iteration (ms): 225407.4 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.51% | FLOPs estimator: workload-aware | logical tokens/s: 74430.4 | valid tokens/s: 74430.4 | physical tokens/s: 75112.9 | loss tokens/s: 58060.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.529981E-01 | lm-global-supervised-tokens: 1.308727E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.617487E-02 | lm-token-buffer-mib: 2.345749E+02 | global_load_balancing_loss: 9.998050E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11186.248 | params rms: 0.060570 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:46:48.098004 | finish at 2026-10-17 19:24:53 + [2026-10-14 05:41:52] iteration 1015/ 2384 | consumed samples: 32480 | elapsed time per iteration (ms): 226993.0 | throughput per GPU (TFLOP/s/GPU): 25.3 | MFU 8.12% | FLOPs estimator: workload-aware | logical tokens/s: 73910.6 | valid tokens/s: 73910.6 | physical tokens/s: 74617.0 | loss tokens/s: 53680.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.015073E-01 | lm-global-supervised-tokens: 1.218516E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.483395E-02 | lm-token-buffer-mib: 2.344784E+02 | global_load_balancing_loss: 9.995416E-01 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11186.216 | params rms: 0.060570 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:19:13.435646 | finish at 2026-10-17 20:01:06 + [2026-10-14 05:45:32] iteration 1016/ 2384 | consumed samples: 32512 | elapsed time per iteration (ms): 219917.7 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.65% | FLOPs estimator: workload-aware | logical tokens/s: 76288.5 | valid tokens/s: 76288.5 | physical tokens/s: 76586.0 | loss tokens/s: 59491.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.909363E-01 | lm-global-supervised-tokens: 1.308332E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.907593E-02 | lm-token-buffer-mib: 2.326350E+02 | global_load_balancing_loss: 9.999465E-01 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11186.185 | params rms: 0.060570 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:34:07.360834 | finish at 2026-10-17 17:19:40 + [2026-10-14 05:49:16] iteration 1017/ 2384 | consumed samples: 32544 | elapsed time per iteration (ms): 223570.2 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.81% | FLOPs estimator: workload-aware | logical tokens/s: 75042.2 | valid tokens/s: 75042.2 | physical tokens/s: 75437.0 | loss tokens/s: 54259.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.543748E-01 | lm-global-supervised-tokens: 1.213073E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.939351E-02 | lm-token-buffer-mib: 2.340385E+02 | global_load_balancing_loss: 1.001353E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11186.150 | params rms: 0.060569 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:53:40.414792 | finish at 2026-10-17 18:42:56 + [2026-10-14 05:53:02] iteration 1018/ 2384 | consumed samples: 32576 | elapsed time per iteration (ms): 226237.4 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.87% | FLOPs estimator: workload-aware | logical tokens/s: 74157.4 | valid tokens/s: 74157.4 | physical tokens/s: 75467.6 | loss tokens/s: 58824.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.745827E-01 | lm-global-supervised-tokens: 1.330831E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.353205E-02 | lm-token-buffer-mib: 2.389044E+02 | global_load_balancing_loss: 1.000632E+00 | loss scale: 1.0 | grad norm: 0.233 | params norm: 11186.111 | params rms: 0.060569 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:50:40.282287 | finish at 2026-10-17 19:43:42 + [2026-10-14 05:56:46] iteration 1019/ 2384 | consumed samples: 32608 | elapsed time per iteration (ms): 224371.8 | throughput per GPU (TFLOP/s/GPU): 26.8 | MFU 8.60% | FLOPs estimator: workload-aware | logical tokens/s: 74774.1 | valid tokens/s: 74774.1 | physical tokens/s: 75723.7 | loss tokens/s: 57561.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.982395E-01 | lm-global-supervised-tokens: 1.291516E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.650662E-02 | lm-token-buffer-mib: 2.331957E+02 | global_load_balancing_loss: 1.000853E+00 | loss scale: 1.0 | grad norm: 0.221 | params norm: 11186.069 | params rms: 0.060569 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:04:27.503997 | finish at 2026-10-17 19:01:14 + [2026-10-14 06:00:52] iteration 1020/ 2384 | consumed samples: 32640 | elapsed time per iteration (ms): 245541.4 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.85% | FLOPs estimator: workload-aware | logical tokens/s: 68327.3 | valid tokens/s: 68327.3 | physical tokens/s: 68826.9 | loss tokens/s: 51961.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.972911E-01 | lm-global-supervised-tokens: 1.275869E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.665528E-02 | lm-token-buffer-mib: 2.340674E+02 | global_load_balancing_loss: 1.000609E+00 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11186.029 | params rms: 0.060569 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:01:58.407101 | finish at 2026-10-18 03:02:50 + [2026-10-14 06:04:31] iteration 1021/ 2384 | consumed samples: 32672 | elapsed time per iteration (ms): 219376.8 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.02% | FLOPs estimator: workload-aware | logical tokens/s: 76476.6 | valid tokens/s: 76476.6 | physical tokens/s: 77091.1 | loss tokens/s: 59351.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.911457E-01 | lm-global-supervised-tokens: 1.302039E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.401810E-02 | lm-token-buffer-mib: 2.341765E+02 | global_load_balancing_loss: 1.001093E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11185.992 | params rms: 0.060569 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:03:30.548260 | finish at 2026-10-17 17:08:02 + [2026-10-14 06:08:13] iteration 1022/ 2384 | consumed samples: 32704 | elapsed time per iteration (ms): 222045.3 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.48% | FLOPs estimator: workload-aware | logical tokens/s: 75557.5 | valid tokens/s: 75557.5 | physical tokens/s: 75995.4 | loss tokens/s: 61945.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.143731E-01 | lm-global-supervised-tokens: 1.375476E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.345806E-02 | lm-token-buffer-mib: 2.334195E+02 | global_load_balancing_loss: 1.000690E+00 | loss scale: 1.0 | grad norm: 0.129 | params norm: 11185.955 | params rms: 0.060568 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:00:25.722314 | finish at 2026-10-17 18:08:39 + [2026-10-14 06:11:56] iteration 1023/ 2384 | consumed samples: 32736 | elapsed time per iteration (ms): 222358.8 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.01% | FLOPs estimator: workload-aware | logical tokens/s: 75451.0 | valid tokens/s: 75451.0 | physical tokens/s: 76689.3 | loss tokens/s: 57142.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.559729E-01 | lm-global-supervised-tokens: 1.270609E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.727331E-02 | lm-token-buffer-mib: 2.342418E+02 | global_load_balancing_loss: 1.001808E+00 | loss scale: 1.0 | grad norm: 0.500 | params norm: 11185.912 | params rms: 0.060568 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:03:50.355266 | finish at 2026-10-17 18:15:46 + [2026-10-14 06:15:37] iteration 1024/ 2384 | consumed samples: 32768 | elapsed time per iteration (ms): 221470.2 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.57% | FLOPs estimator: workload-aware | logical tokens/s: 75753.8 | valid tokens/s: 75753.8 | physical tokens/s: 76484.7 | loss tokens/s: 54804.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.801921E-01 | lm-global-supervised-tokens: 1.213748E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.048733E-02 | lm-token-buffer-mib: 2.357825E+02 | global_load_balancing_loss: 1.000778E+00 | loss scale: 1.0 | grad norm: 0.291 | params norm: 11185.869 | params rms: 0.060568 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:39:59.471760 | finish at 2026-10-17 17:55:37 + [2026-10-14 06:19:21] iteration 1025/ 2384 | consumed samples: 32800 | elapsed time per iteration (ms): 223402.4 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.20% | FLOPs estimator: workload-aware | logical tokens/s: 75098.5 | valid tokens/s: 75098.5 | physical tokens/s: 75795.1 | loss tokens/s: 56058.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.238561E-01 | lm-global-supervised-tokens: 1.252365E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.339525E-02 | lm-token-buffer-mib: 2.351872E+02 | global_load_balancing_loss: 9.997215E-01 | loss scale: 1.0 | grad norm: 0.227 | params norm: 11185.827 | params rms: 0.060568 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:20:03.796821 | finish at 2026-10-17 18:39:24 + [2026-10-14 06:23:08] iteration 1026/ 2384 | consumed samples: 32832 | elapsed time per iteration (ms): 226942.1 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.24% | FLOPs estimator: workload-aware | logical tokens/s: 73927.2 | valid tokens/s: 73927.2 | physical tokens/s: 75348.9 | loss tokens/s: 55100.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.150191E-01 | lm-global-supervised-tokens: 1.250456E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.310108E-02 | lm-token-buffer-mib: 2.389412E+02 | global_load_balancing_loss: 9.999090E-01 | loss scale: 1.0 | grad norm: 0.128 | params norm: 11185.786 | params rms: 0.060567 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:36:27.316176 | finish at 2026-10-17 19:59:35 + [2026-10-14 06:26:53] iteration 1027/ 2384 | consumed samples: 32864 | elapsed time per iteration (ms): 225329.9 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.93% | FLOPs estimator: workload-aware | logical tokens/s: 74456.2 | valid tokens/s: 74456.2 | physical tokens/s: 75734.9 | loss tokens/s: 56109.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.240507E-01 | lm-global-supervised-tokens: 1.264306E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.510544E-02 | lm-token-buffer-mib: 2.355058E+02 | global_load_balancing_loss: 1.000838E+00 | loss scale: 1.0 | grad norm: 0.245 | params norm: 11185.747 | params rms: 0.060567 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:56:12.695365 | finish at 2026-10-17 19:23:06 + [2026-10-14 06:30:36] iteration 1028/ 2384 | consumed samples: 32896 | elapsed time per iteration (ms): 223256.3 | throughput per GPU (TFLOP/s/GPU): 24.2 | MFU 7.75% | FLOPs estimator: workload-aware | logical tokens/s: 75147.7 | valid tokens/s: 75147.7 | physical tokens/s: 75409.8 | loss tokens/s: 53459.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.143726E-01 | lm-global-supervised-tokens: 1.193519E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.522509E-02 | lm-token-buffer-mib: 2.330376E+02 | global_load_balancing_loss: 9.988275E-01 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11185.703 | params rms: 0.060567 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:05:35.548582 | finish at 2026-10-17 18:36:12 + [2026-10-14 06:34:19] iteration 1029/ 2384 | consumed samples: 32928 | elapsed time per iteration (ms): 223021.9 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.99% | FLOPs estimator: workload-aware | logical tokens/s: 75226.7 | valid tokens/s: 75226.7 | physical tokens/s: 75820.8 | loss tokens/s: 57398.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.647581E-01 | lm-global-supervised-tokens: 1.280114E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.915837E-02 | lm-token-buffer-mib: 2.343493E+02 | global_load_balancing_loss: 9.994837E-01 | loss scale: 1.0 | grad norm: 0.154 | params norm: 11185.657 | params rms: 0.060567 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:56:34.716414 | finish at 2026-10-17 18:30:54 + [2026-10-14 06:38:25] iteration 1030/ 2384 | consumed samples: 32960 | elapsed time per iteration (ms): 245956.2 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.08% | FLOPs estimator: workload-aware | logical tokens/s: 68212.1 | valid tokens/s: 68212.1 | physical tokens/s: 69152.3 | loss tokens/s: 51602.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.078554E-01 | lm-global-supervised-tokens: 1.269193E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.650588E-02 | lm-token-buffer-mib: 2.343695E+02 | global_load_balancing_loss: 1.000763E+00 | loss scale: 1.0 | grad norm: 0.211 | params norm: 11185.612 | params rms: 0.060566 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:30:24.675597 | finish at 2026-10-18 03:08:50 + [2026-10-14 06:42:08] iteration 1031/ 2384 | consumed samples: 32992 | elapsed time per iteration (ms): 222666.9 | throughput per GPU (TFLOP/s/GPU): 20.2 | MFU 6.47% | FLOPs estimator: workload-aware | logical tokens/s: 75346.6 | valid tokens/s: 75346.6 | physical tokens/s: 76478.0 | loss tokens/s: 59216.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.369941E-01 | lm-global-supervised-tokens: 1.318552E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.695010E-02 | lm-token-buffer-mib: 2.348992E+02 | global_load_balancing_loss: 1.000023E+00 | loss scale: 1.0 | grad norm: 0.243 | params norm: 11185.567 | params rms: 0.060566 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:41:08.267849 | finish at 2026-10-17 18:23:16 + [2026-10-14 06:45:52] iteration 1032/ 2384 | consumed samples: 33024 | elapsed time per iteration (ms): 224565.1 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.23% | FLOPs estimator: workload-aware | logical tokens/s: 74709.8 | valid tokens/s: 74709.8 | physical tokens/s: 75705.0 | loss tokens/s: 59170.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.732852E-01 | lm-global-supervised-tokens: 1.328760E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.674205E-02 | lm-token-buffer-mib: 2.361547E+02 | global_load_balancing_loss: 1.000647E+00 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11185.523 | params rms: 0.060566 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:20:12.011271 | finish at 2026-10-17 19:06:04 + [2026-10-14 06:49:34] iteration 1033/ 2384 | consumed samples: 33056 | elapsed time per iteration (ms): 221096.6 | throughput per GPU (TFLOP/s/GPU): 24.7 | MFU 7.93% | FLOPs estimator: workload-aware | logical tokens/s: 75881.6 | valid tokens/s: 75881.6 | physical tokens/s: 76508.5 | loss tokens/s: 56206.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.467679E-01 | lm-global-supervised-tokens: 1.242716E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.598163E-02 | lm-token-buffer-mib: 2.352909E+02 | global_load_balancing_loss: 1.001903E+00 | loss scale: 1.0 | grad norm: 0.116 | params norm: 11185.475 | params rms: 0.060566 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:58:21.518593 | finish at 2026-10-17 17:47:55 + [2026-10-14 06:53:15] iteration 1034/ 2384 | consumed samples: 33088 | elapsed time per iteration (ms): 221860.5 | throughput per GPU (TFLOP/s/GPU): 23.4 | MFU 7.50% | FLOPs estimator: workload-aware | logical tokens/s: 75620.5 | valid tokens/s: 75620.5 | physical tokens/s: 76196.0 | loss tokens/s: 54768.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.640628E-01 | lm-global-supervised-tokens: 1.215085E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.189216E-02 | lm-token-buffer-mib: 2.336559E+02 | global_load_balancing_loss: 1.001376E+00 | loss scale: 1.0 | grad norm: 0.171 | params norm: 11185.427 | params rms: 0.060565 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:11:51.720192 | finish at 2026-10-17 18:05:07 + [2026-10-14 06:56:58] iteration 1035/ 2384 | consumed samples: 33120 | elapsed time per iteration (ms): 222878.1 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.55% | FLOPs estimator: workload-aware | logical tokens/s: 75275.3 | valid tokens/s: 75275.3 | physical tokens/s: 76032.5 | loss tokens/s: 58100.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.620819E-01 | lm-global-supervised-tokens: 1.294934E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.673533E-02 | lm-token-buffer-mib: 2.342624E+02 | global_load_balancing_loss: 1.000859E+00 | loss scale: 1.0 | grad norm: 0.177 | params norm: 11185.382 | params rms: 0.060565 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:31:02.536206 | finish at 2026-10-17 18:28:01 + [2026-10-14 07:00:40] iteration 1036/ 2384 | consumed samples: 33152 | elapsed time per iteration (ms): 221812.2 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.56% | FLOPs estimator: workload-aware | logical tokens/s: 75636.9 | valid tokens/s: 75636.9 | physical tokens/s: 76816.5 | loss tokens/s: 61639.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.422580E-01 | lm-global-supervised-tokens: 1.367234E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.731673E-02 | lm-token-buffer-mib: 2.345625E+02 | global_load_balancing_loss: 1.000426E+00 | loss scale: 1.0 | grad norm: 0.158 | params norm: 11185.341 | params rms: 0.060565 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:03:22.903865 | finish at 2026-10-17 18:04:03 + [2026-10-14 07:04:24] iteration 1037/ 2384 | consumed samples: 33184 | elapsed time per iteration (ms): 223568.0 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.95% | FLOPs estimator: workload-aware | logical tokens/s: 75042.9 | valid tokens/s: 75042.9 | physical tokens/s: 76182.4 | loss tokens/s: 55379.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.065056E-01 | lm-global-supervised-tokens: 1.238099E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.362455E-02 | lm-token-buffer-mib: 2.353853E+02 | global_load_balancing_loss: 9.999490E-01 | loss scale: 1.0 | grad norm: 0.199 | params norm: 11185.300 | params rms: 0.060565 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:39:06.039583 | finish at 2026-10-17 18:43:30 + [2026-10-14 07:08:04] iteration 1038/ 2384 | consumed samples: 33216 | elapsed time per iteration (ms): 220652.0 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.82% | FLOPs estimator: workload-aware | logical tokens/s: 76034.5 | valid tokens/s: 76034.5 | physical tokens/s: 76963.5 | loss tokens/s: 58249.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.636015E-01 | lm-global-supervised-tokens: 1.285295E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.771854E-02 | lm-token-buffer-mib: 2.346358E+02 | global_load_balancing_loss: 9.991902E-01 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11185.259 | params rms: 0.060565 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:29:57.603486 | finish at 2026-10-17 17:38:02 + [2026-10-14 07:11:46] iteration 1039/ 2384 | consumed samples: 33248 | elapsed time per iteration (ms): 221470.9 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.29% | FLOPs estimator: workload-aware | logical tokens/s: 75753.6 | valid tokens/s: 75753.6 | physical tokens/s: 76422.3 | loss tokens/s: 56748.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.483727E-01 | lm-global-supervised-tokens: 1.256809E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.243236E-02 | lm-token-buffer-mib: 2.343089E+02 | global_load_balancing_loss: 1.000578E+00 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11185.218 | params rms: 0.060564 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:44:38.394570 | finish at 2026-10-17 17:56:24 + [2026-10-14 07:15:51] iteration 1040/ 2384 | consumed samples: 33280 | elapsed time per iteration (ms): 245490.5 | throughput per GPU (TFLOP/s/GPU): 17.5 | MFU 5.62% | FLOPs estimator: workload-aware | logical tokens/s: 68341.6 | valid tokens/s: 68341.6 | physical tokens/s: 69446.2 | loss tokens/s: 52644.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.842547E-01 | lm-global-supervised-tokens: 1.292366E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.079705E-02 | lm-token-buffer-mib: 2.351038E+02 | global_load_balancing_loss: 9.992430E-01 | loss scale: 1.0 | grad norm: 0.109 | params norm: 11185.176 | params rms: 0.060564 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 19:38:59.284515 | finish at 2026-10-18 02:54:51 +times across ranks (ms): + save-checkpoint: + rank 0: 26818.75 + rank 1: 26818.76 + rank 2: 26818.84 + rank 3: 26818.78 + rank 4: 26818.75 + rank 5: 26818.77 + rank 6: 26818.76 + rank 7: 26818.76 + rank 8: 26817.64 + rank 9: 26817.65 + rank 10: 26817.70 + rank 11: 26817.64 + rank 12: 26817.63 + rank 13: 26817.63 + rank 14: 26817.63 + rank 15: 26817.48 + rank 16: 26818.80 + rank 17: 26818.76 + rank 18: 26818.79 + rank 19: 26818.76 + rank 20: 26818.75 + rank 21: 26818.77 + rank 22: 26818.82 + rank 23: 26818.73 + rank 24: 26818.61 + rank 25: 26818.64 + rank 26: 26818.58 + rank 27: 26818.68 + rank 28: 26818.57 + rank 29: 26818.60 + rank 30: 26818.70 + rank 31: 26818.71 + rank 32: 26818.07 + rank 33: 26818.05 + rank 34: 26818.11 + rank 35: 26818.07 + rank 36: 26818.02 + rank 37: 26818.09 + rank 38: 26818.05 + rank 39: 26818.19 + rank 40: 26819.45 + rank 41: 26819.44 + rank 42: 26819.44 + rank 43: 26819.45 + rank 44: 26819.42 + rank 45: 26819.46 + rank 46: 26819.43 + rank 47: 26819.42 + rank 48: 26818.82 + rank 49: 26818.86 + rank 50: 26818.85 + rank 51: 26818.82 + rank 52: 26818.82 + rank 53: 26818.86 + rank 54: 26818.82 + rank 55: 26818.84 + rank 56: 26818.65 + rank 57: 26818.69 + rank 58: 26818.70 + rank 59: 26818.71 + rank 60: 26818.70 + rank 61: 26818.72 + rank 62: 26818.67 + rank 63: 26818.66 + [2026-10-14 07:20:23] iteration 1041/ 2384 | consumed samples: 33312 | elapsed time per iteration (ms): 245377.9 | throughput per GPU (TFLOP/s/GPU): 19.6 | MFU 6.28% | FLOPs estimator: workload-aware | logical tokens/s: 68372.7 | valid tokens/s: 68372.7 | physical tokens/s: 69333.8 | loss tokens/s: 50535.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.171594E-01 | lm-global-supervised-tokens: 1.240026E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.897097E-02 | lm-token-buffer-mib: 2.357247E+02 | global_load_balancing_loss: 9.994258E-01 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11185.135 | params rms: 0.060564 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 19:32:22.581344 | finish at 2026-10-18 02:52:46 + [2026-10-14 07:24:10] iteration 1042/ 2384 | consumed samples: 33344 | elapsed time per iteration (ms): 226570.5 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.34% | FLOPs estimator: workload-aware | logical tokens/s: 74048.4 | valid tokens/s: 74048.4 | physical tokens/s: 74518.0 | loss tokens/s: 56024.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.898151E-01 | lm-global-supervised-tokens: 1.269338E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.359328E-02 | lm-token-buffer-mib: 2.335364E+02 | global_load_balancing_loss: 1.001005E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11185.092 | params rms: 0.060564 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:27:37.656616 | finish at 2026-10-17 19:51:48 + [2026-10-14 07:27:53] iteration 1043/ 2384 | consumed samples: 33376 | elapsed time per iteration (ms): 222525.4 | throughput per GPU (TFLOP/s/GPU): 25.2 | MFU 8.09% | FLOPs estimator: workload-aware | logical tokens/s: 75394.5 | valid tokens/s: 75394.5 | physical tokens/s: 75806.7 | loss tokens/s: 56960.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.219561E-01 | lm-global-supervised-tokens: 1.267517E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.530527E-02 | lm-token-buffer-mib: 2.334686E+02 | global_load_balancing_loss: 1.000007E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11185.048 | params rms: 0.060563 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:53:26.577283 | finish at 2026-10-17 18:21:19 + [2026-10-14 07:31:32] iteration 1044/ 2384 | consumed samples: 33408 | elapsed time per iteration (ms): 219581.8 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.78% | FLOPs estimator: workload-aware | logical tokens/s: 76405.1 | valid tokens/s: 76405.1 | physical tokens/s: 77217.6 | loss tokens/s: 55580.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.021168E-01 | lm-global-supervised-tokens: 1.220436E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.478486E-02 | lm-token-buffer-mib: 2.340067E+02 | global_load_balancing_loss: 1.001184E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11185.006 | params rms: 0.060563 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:43:59.651594 | finish at 2026-10-17 17:15:32 + [2026-10-14 07:35:14] iteration 1045/ 2384 | consumed samples: 33440 | elapsed time per iteration (ms): 221581.1 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.05% | FLOPs estimator: workload-aware | logical tokens/s: 75715.7 | valid tokens/s: 75715.7 | physical tokens/s: 76726.8 | loss tokens/s: 58197.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.139081E-01 | lm-global-supervised-tokens: 1.289556E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.582325E-02 | lm-token-buffer-mib: 2.346768E+02 | global_load_balancing_loss: 1.001889E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11184.967 | params rms: 0.060563 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:24:57.153219 | finish at 2026-10-17 18:00:11 + [2026-10-14 07:38:54] iteration 1046/ 2384 | consumed samples: 33472 | elapsed time per iteration (ms): 220359.1 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.02% | FLOPs estimator: workload-aware | logical tokens/s: 76135.8 | valid tokens/s: 76135.8 | physical tokens/s: 76932.5 | loss tokens/s: 58664.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.697501E-01 | lm-global-supervised-tokens: 1.292723E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.725212E-02 | lm-token-buffer-mib: 2.341048E+02 | global_load_balancing_loss: 1.001663E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11184.933 | params rms: 0.060563 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:54:00.492208 | finish at 2026-10-17 17:32:55 + [2026-10-14 07:42:35] iteration 1047/ 2384 | consumed samples: 33504 | elapsed time per iteration (ms): 220631.5 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.18% | FLOPs estimator: workload-aware | logical tokens/s: 76041.7 | valid tokens/s: 76041.7 | physical tokens/s: 76466.5 | loss tokens/s: 52119.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.160988E-01 | lm-global-supervised-tokens: 1.149926E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.609233E-02 | lm-token-buffer-mib: 2.325928E+02 | global_load_balancing_loss: 1.001089E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11184.896 | params rms: 0.060563 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:56:24.359497 | finish at 2026-10-17 17:38:59 + [2026-10-14 07:46:16] iteration 1048/ 2384 | consumed samples: 33536 | elapsed time per iteration (ms): 221712.3 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.51% | FLOPs estimator: workload-aware | logical tokens/s: 75671.0 | valid tokens/s: 75671.0 | physical tokens/s: 76186.4 | loss tokens/s: 56828.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.861825E-01 | lm-global-supervised-tokens: 1.259963E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.250518E-02 | lm-token-buffer-mib: 2.345108E+02 | global_load_balancing_loss: 9.996719E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11184.858 | params rms: 0.060562 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:16:47.581919 | finish at 2026-10-17 18:03:04 + [2026-10-14 07:49:59] iteration 1049/ 2384 | consumed samples: 33568 | elapsed time per iteration (ms): 222675.8 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.83% | FLOPs estimator: workload-aware | logical tokens/s: 75343.5 | valid tokens/s: 75343.5 | physical tokens/s: 76066.0 | loss tokens/s: 60211.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.965376E-01 | lm-global-supervised-tokens: 1.340758E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.681384E-02 | lm-token-buffer-mib: 2.336336E+02 | global_load_balancing_loss: 9.990115E-01 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11184.815 | params rms: 0.060562 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:34:32.167332 | finish at 2026-10-17 18:24:31 + [2026-10-14 07:54:03] iteration 1050/ 2384 | consumed samples: 33600 | elapsed time per iteration (ms): 244391.9 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.62% | FLOPs estimator: workload-aware | logical tokens/s: 68648.7 | valid tokens/s: 68648.7 | physical tokens/s: 69805.3 | loss tokens/s: 52590.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.664661E-01 | lm-global-supervised-tokens: 1.285262E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.015629E-02 | lm-token-buffer-mib: 2.336388E+02 | global_load_balancing_loss: 9.989833E-01 | loss scale: 1.0 | grad norm: 0.119 | params norm: 11184.774 | params rms: 0.060562 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:33:38.787686 | finish at 2026-10-18 02:27:42 + [2026-10-14 07:57:50] iteration 1051/ 2384 | consumed samples: 33632 | elapsed time per iteration (ms): 226088.1 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.24% | FLOPs estimator: workload-aware | logical tokens/s: 74206.5 | valid tokens/s: 74206.5 | physical tokens/s: 75331.8 | loss tokens/s: 58939.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.369125E-01 | lm-global-supervised-tokens: 1.332561E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.815994E-02 | lm-token-buffer-mib: 2.361445E+02 | global_load_balancing_loss: 9.972283E-01 | loss scale: 1.0 | grad norm: 0.110 | params norm: 11184.732 | params rms: 0.060562 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:42:55.407050 | finish at 2026-10-17 19:40:45 + [2026-10-14 08:01:31] iteration 1052/ 2384 | consumed samples: 33664 | elapsed time per iteration (ms): 221582.7 | throughput per GPU (TFLOP/s/GPU): 23.2 | MFU 7.43% | FLOPs estimator: workload-aware | logical tokens/s: 75715.1 | valid tokens/s: 75715.1 | physical tokens/s: 76221.1 | loss tokens/s: 56338.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.587636E-01 | lm-global-supervised-tokens: 1.248369E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.501899E-02 | lm-token-buffer-mib: 2.341118E+02 | global_load_balancing_loss: 1.000156E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11184.691 | params rms: 0.060561 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:59:08.158324 | finish at 2026-10-17 18:00:39 + [2026-10-14 08:05:13] iteration 1053/ 2384 | consumed samples: 33696 | elapsed time per iteration (ms): 221760.8 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.05% | FLOPs estimator: workload-aware | logical tokens/s: 75654.5 | valid tokens/s: 75654.5 | physical tokens/s: 76220.2 | loss tokens/s: 57296.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.128121E-01 | lm-global-supervised-tokens: 1.270605E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.563671E-02 | lm-token-buffer-mib: 2.335134E+02 | global_load_balancing_loss: 1.001859E+00 | loss scale: 1.0 | grad norm: 0.257 | params norm: 11184.650 | params rms: 0.060561 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:59:23.620204 | finish at 2026-10-17 18:04:37 + [2026-10-14 08:08:57] iteration 1054/ 2384 | consumed samples: 33728 | elapsed time per iteration (ms): 223807.0 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.72% | FLOPs estimator: workload-aware | logical tokens/s: 74962.7 | valid tokens/s: 74962.7 | physical tokens/s: 75588.9 | loss tokens/s: 57731.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.919155E-01 | lm-global-supervised-tokens: 1.292081E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.951456E-02 | lm-token-buffer-mib: 2.331489E+02 | global_load_balancing_loss: 1.003019E+00 | loss scale: 1.0 | grad norm: 0.166 | params norm: 11184.610 | params rms: 0.060561 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:41:03.272796 | finish at 2026-10-17 18:50:00 + [2026-10-14 08:12:39] iteration 1055/ 2384 | consumed samples: 33760 | elapsed time per iteration (ms): 222174.6 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.95% | FLOPs estimator: workload-aware | logical tokens/s: 75513.6 | valid tokens/s: 75513.6 | physical tokens/s: 76080.5 | loss tokens/s: 58565.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.035038E-01 | lm-global-supervised-tokens: 1.301172E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.636132E-02 | lm-token-buffer-mib: 2.327606E+02 | global_load_balancing_loss: 1.002303E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11184.571 | params rms: 0.060561 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:01:10.099966 | finish at 2026-10-17 18:13:49 + [2026-10-14 08:16:24] iteration 1056/ 2384 | consumed samples: 33792 | elapsed time per iteration (ms): 224880.0 | throughput per GPU (TFLOP/s/GPU): 18.1 | MFU 5.79% | FLOPs estimator: workload-aware | logical tokens/s: 74605.0 | valid tokens/s: 74605.0 | physical tokens/s: 75894.4 | loss tokens/s: 57813.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.624006E-01 | lm-global-supervised-tokens: 1.300106E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.838530E-02 | lm-token-buffer-mib: 2.365938E+02 | global_load_balancing_loss: 1.001307E+00 | loss scale: 1.0 | grad norm: 0.152 | params norm: 11184.530 | params rms: 0.060561 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:57:20.624638 | finish at 2026-10-17 19:13:44 + [2026-10-14 08:20:06] iteration 1057/ 2384 | consumed samples: 33824 | elapsed time per iteration (ms): 222025.5 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 7.00% | FLOPs estimator: workload-aware | logical tokens/s: 75564.2 | valid tokens/s: 75564.2 | physical tokens/s: 76438.6 | loss tokens/s: 54850.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.084721E-01 | lm-global-supervised-tokens: 1.217818E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.798930E-02 | lm-token-buffer-mib: 2.353470E+02 | global_load_balancing_loss: 9.995781E-01 | loss scale: 1.0 | grad norm: 0.146 | params norm: 11184.492 | params rms: 0.060560 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:50:27.865471 | finish at 2026-10-17 18:10:34 + [2026-10-14 08:23:48] iteration 1058/ 2384 | consumed samples: 33856 | elapsed time per iteration (ms): 221973.3 | throughput per GPU (TFLOP/s/GPU): 19.2 | MFU 6.14% | FLOPs estimator: workload-aware | logical tokens/s: 75582.1 | valid tokens/s: 75582.1 | physical tokens/s: 76034.5 | loss tokens/s: 55995.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.098938E-01 | lm-global-supervised-tokens: 1.242958E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.493310E-02 | lm-token-buffer-mib: 2.328396E+02 | global_load_balancing_loss: 9.993728E-01 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11184.456 | params rms: 0.060560 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:45:36.530332 | finish at 2026-10-17 18:09:24 + [2026-10-14 08:27:29] iteration 1059/ 2384 | consumed samples: 33888 | elapsed time per iteration (ms): 221410.2 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.40% | FLOPs estimator: workload-aware | logical tokens/s: 75774.2 | valid tokens/s: 75774.2 | physical tokens/s: 76593.8 | loss tokens/s: 57244.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.301282E-01 | lm-global-supervised-tokens: 1.267454E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.078365E-02 | lm-token-buffer-mib: 2.340298E+02 | global_load_balancing_loss: 9.986334E-01 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11184.421 | params rms: 0.060560 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:29:28.499362 | finish at 2026-10-17 17:56:58 + [2026-10-14 08:31:30] iteration 1060/ 2384 | consumed samples: 33920 | elapsed time per iteration (ms): 240783.7 | throughput per GPU (TFLOP/s/GPU): 19.7 | MFU 6.31% | FLOPs estimator: workload-aware | logical tokens/s: 69677.5 | valid tokens/s: 69677.5 | physical tokens/s: 70333.0 | loss tokens/s: 54930.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.419765E-01 | lm-global-supervised-tokens: 1.322636E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.271355E-02 | lm-token-buffer-mib: 2.338602E+02 | global_load_balancing_loss: 9.987599E-01 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11184.387 | params rms: 0.060560 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:33:17.595742 | finish at 2026-10-18 01:04:48 + [2026-10-14 08:35:14] iteration 1061/ 2384 | consumed samples: 33952 | elapsed time per iteration (ms): 223618.9 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.03% | FLOPs estimator: workload-aware | logical tokens/s: 75025.8 | valid tokens/s: 75025.8 | physical tokens/s: 75639.5 | loss tokens/s: 56391.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.287422E-01 | lm-global-supervised-tokens: 1.261022E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.613005E-02 | lm-token-buffer-mib: 2.350333E+02 | global_load_balancing_loss: 9.989091E-01 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11184.353 | params rms: 0.060560 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:10:47.799418 | finish at 2026-10-17 18:46:01 + [2026-10-14 08:38:56] iteration 1062/ 2384 | consumed samples: 33984 | elapsed time per iteration (ms): 222627.8 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.90% | FLOPs estimator: workload-aware | logical tokens/s: 75359.8 | valid tokens/s: 75359.8 | physical tokens/s: 76151.9 | loss tokens/s: 55511.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.520961E-01 | lm-global-supervised-tokens: 1.235835E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.994098E-02 | lm-token-buffer-mib: 2.351330E+02 | global_load_balancing_loss: 1.000919E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11184.320 | params rms: 0.060559 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:45:13.966083 | finish at 2026-10-17 18:24:10 + [2026-10-14 08:42:39] iteration 1063/ 2384 | consumed samples: 34016 | elapsed time per iteration (ms): 222960.9 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.03% | FLOPs estimator: workload-aware | logical tokens/s: 75247.2 | valid tokens/s: 75247.2 | physical tokens/s: 75900.7 | loss tokens/s: 58780.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.916618E-01 | lm-global-supervised-tokens: 1.310569E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.033174E-02 | lm-token-buffer-mib: 2.337688E+02 | global_load_balancing_loss: 1.000459E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11184.287 | params rms: 0.060559 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:48:51.331668 | finish at 2026-10-17 18:31:31 + [2026-10-14 08:46:19] iteration 1064/ 2384 | consumed samples: 34048 | elapsed time per iteration (ms): 219882.6 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.01% | FLOPs estimator: workload-aware | logical tokens/s: 76300.7 | valid tokens/s: 76300.7 | physical tokens/s: 76717.0 | loss tokens/s: 56713.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.193625E-01 | lm-global-supervised-tokens: 1.247024E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.649742E-02 | lm-token-buffer-mib: 2.335021E+02 | global_load_balancing_loss: 1.002096E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11184.250 | params rms: 0.060559 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:37:24.993696 | finish at 2026-10-17 17:23:44 + [2026-10-14 08:50:59] iteration 1065/ 2384 | consumed samples: 34080 | elapsed time per iteration (ms): 280003.6 | throughput per GPU (TFLOP/s/GPU): 14.8 | MFU 4.75% | FLOPs estimator: workload-aware | logical tokens/s: 58216.6 | valid tokens/s: 58216.6 | physical tokens/s: 59197.2 | loss tokens/s: 44958.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.537838E-01 | lm-global-supervised-tokens: 1.258857E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 2.155370E-01 | lm-token-buffer-mib: 2.219308E+02 | global_load_balancing_loss: 1.001261E+00 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11184.217 | params rms: 0.060559 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:35:24.702331 | finish at 2026-10-18 15:26:24 + [2026-10-14 08:54:40] iteration 1066/ 2384 | consumed samples: 34112 | elapsed time per iteration (ms): 221340.7 | throughput per GPU (TFLOP/s/GPU): 25.2 | MFU 8.07% | FLOPs estimator: workload-aware | logical tokens/s: 75798.0 | valid tokens/s: 75798.0 | physical tokens/s: 76354.2 | loss tokens/s: 56518.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.943617E-01 | lm-global-supervised-tokens: 1.250988E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.491351E-02 | lm-token-buffer-mib: 2.326044E+02 | global_load_balancing_loss: 1.000987E+00 | loss scale: 1.0 | grad norm: 0.115 | params norm: 11184.182 | params rms: 0.060559 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:02:07.074535 | finish at 2026-10-17 17:56:48 + [2026-10-14 08:58:27] iteration 1067/ 2384 | consumed samples: 34144 | elapsed time per iteration (ms): 226481.7 | throughput per GPU (TFLOP/s/GPU): 19.5 | MFU 6.25% | FLOPs estimator: workload-aware | logical tokens/s: 74077.5 | valid tokens/s: 74077.5 | physical tokens/s: 75334.8 | loss tokens/s: 54339.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.325812E-01 | lm-global-supervised-tokens: 1.230687E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.506657E-02 | lm-token-buffer-mib: 2.359335E+02 | global_load_balancing_loss: 1.001762E+00 | loss scale: 1.0 | grad norm: 0.141 | params norm: 11184.146 | params rms: 0.060559 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:51:16.400082 | finish at 2026-10-17 19:49:43 + [2026-10-14 09:02:12] iteration 1068/ 2384 | consumed samples: 34176 | elapsed time per iteration (ms): 224954.9 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.56% | FLOPs estimator: workload-aware | logical tokens/s: 74580.1 | valid tokens/s: 74580.1 | physical tokens/s: 74965.3 | loss tokens/s: 58601.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.572635E-01 | lm-global-supervised-tokens: 1.318273E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.399136E-02 | lm-token-buffer-mib: 2.326167E+02 | global_load_balancing_loss: 1.000545E+00 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11184.113 | params rms: 0.060558 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:14:00.587879 | finish at 2026-10-17 19:16:12 + [2026-10-14 09:05:53] iteration 1069/ 2384 | consumed samples: 34208 | elapsed time per iteration (ms): 220681.1 | throughput per GPU (TFLOP/s/GPU): 24.1 | MFU 7.72% | FLOPs estimator: workload-aware | logical tokens/s: 76024.5 | valid tokens/s: 76024.5 | physical tokens/s: 76367.8 | loss tokens/s: 54361.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.910494E-01 | lm-global-supervised-tokens: 1.199654E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.384868E-02 | lm-token-buffer-mib: 2.327961E+02 | global_load_balancing_loss: 9.996989E-01 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11184.081 | params rms: 0.060558 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:36:35.664855 | finish at 2026-10-17 17:42:28 + [2026-10-14 09:09:55] iteration 1070/ 2384 | consumed samples: 34240 | elapsed time per iteration (ms): 242159.8 | throughput per GPU (TFLOP/s/GPU): 17.3 | MFU 5.55% | FLOPs estimator: workload-aware | logical tokens/s: 69281.6 | valid tokens/s: 69281.6 | physical tokens/s: 69821.0 | loss tokens/s: 50069.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.156121E-01 | lm-global-supervised-tokens: 1.212478E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.768966E-02 | lm-token-buffer-mib: 2.337632E+02 | global_load_balancing_loss: 9.985169E-01 | loss scale: 1.0 | grad norm: 0.105 | params norm: 11184.048 | params rms: 0.060558 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:23:17.964738 | finish at 2026-10-18 01:33:13 + [2026-10-14 09:13:38] iteration 1071/ 2384 | consumed samples: 34272 | elapsed time per iteration (ms): 223686.7 | throughput per GPU (TFLOP/s/GPU): 23.8 | MFU 7.63% | FLOPs estimator: workload-aware | logical tokens/s: 75003.1 | valid tokens/s: 75003.1 | physical tokens/s: 75473.6 | loss tokens/s: 58873.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.338374E-01 | lm-global-supervised-tokens: 1.316927E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.545688E-02 | lm-token-buffer-mib: 2.336231E+02 | global_load_balancing_loss: 9.989818E-01 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11184.016 | params rms: 0.060558 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:35:00.661030 | finish at 2026-10-17 18:48:39 + [2026-10-14 09:17:20] iteration 1072/ 2384 | consumed samples: 34304 | elapsed time per iteration (ms): 221665.0 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.26% | FLOPs estimator: workload-aware | logical tokens/s: 75687.1 | valid tokens/s: 75687.1 | physical tokens/s: 76608.6 | loss tokens/s: 57850.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.542792E-01 | lm-global-supervised-tokens: 1.282350E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.454102E-02 | lm-token-buffer-mib: 2.356472E+02 | global_load_balancing_loss: 9.993902E-01 | loss scale: 1.0 | grad norm: 0.127 | params norm: 11183.983 | params rms: 0.060558 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:47:04.468689 | finish at 2026-10-17 18:04:25 + [2026-10-14 09:21:02] iteration 1073/ 2384 | consumed samples: 34336 | elapsed time per iteration (ms): 221588.9 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.46% | FLOPs estimator: workload-aware | logical tokens/s: 75713.2 | valid tokens/s: 75713.2 | physical tokens/s: 76144.2 | loss tokens/s: 57921.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.786572E-01 | lm-global-supervised-tokens: 1.283465E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.900428E-02 | lm-token-buffer-mib: 2.330591E+02 | global_load_balancing_loss: 1.000035E+00 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11183.948 | params rms: 0.060557 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:41:42.995193 | finish at 2026-10-17 18:02:45 + [2026-10-14 09:24:45] iteration 1074/ 2384 | consumed samples: 34368 | elapsed time per iteration (ms): 223123.7 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.71% | FLOPs estimator: workload-aware | logical tokens/s: 75192.4 | valid tokens/s: 75192.4 | physical tokens/s: 76106.9 | loss tokens/s: 57700.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.437517E-01 | lm-global-supervised-tokens: 1.287444E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.253870E-02 | lm-token-buffer-mib: 2.352391E+02 | global_load_balancing_loss: 1.000300E+00 | loss scale: 1.0 | grad norm: 0.184 | params norm: 11183.914 | params rms: 0.060557 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:11:32.107465 | finish at 2026-10-17 18:36:17 + [2026-10-14 09:28:25] iteration 1075/ 2384 | consumed samples: 34400 | elapsed time per iteration (ms): 220191.1 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.76% | FLOPs estimator: workload-aware | logical tokens/s: 76193.8 | valid tokens/s: 76193.7 | physical tokens/s: 76628.1 | loss tokens/s: 54651.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.761000E-01 | lm-global-supervised-tokens: 1.203373E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.661140E-02 | lm-token-buffer-mib: 2.326800E+02 | global_load_balancing_loss: 1.000742E+00 | loss scale: 1.0 | grad norm: 0.467 | params norm: 11183.879 | params rms: 0.060557 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:03:50.090761 | finish at 2026-10-17 17:32:15 + [2026-10-14 09:32:06] iteration 1076/ 2384 | consumed samples: 34432 | elapsed time per iteration (ms): 221114.2 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.51% | FLOPs estimator: workload-aware | logical tokens/s: 75875.8 | valid tokens/s: 75875.7 | physical tokens/s: 76558.3 | loss tokens/s: 60238.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.163088E-01 | lm-global-supervised-tokens: 1.331961E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.812370E-02 | lm-token-buffer-mib: 2.347467E+02 | global_load_balancing_loss: 1.001008E+00 | loss scale: 1.0 | grad norm: 0.286 | params norm: 11183.844 | params rms: 0.060557 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:20:17.339759 | finish at 2026-10-17 17:52:23 + [2026-10-14 09:35:54] iteration 1077/ 2384 | consumed samples: 34464 | elapsed time per iteration (ms): 228122.0 | throughput per GPU (TFLOP/s/GPU): 23.2 | MFU 7.44% | FLOPs estimator: workload-aware | logical tokens/s: 73544.8 | valid tokens/s: 73544.8 | physical tokens/s: 74800.4 | loss tokens/s: 56559.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.506998E-01 | lm-global-supervised-tokens: 1.290248E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.305271E-02 | lm-token-buffer-mib: 2.363848E+02 | global_load_balancing_loss: 1.002874E+00 | loss scale: 1.0 | grad norm: 0.688 | params norm: 11183.812 | params rms: 0.060557 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:49:15.446182 | finish at 2026-10-17 20:25:10 + [2026-10-14 09:39:41] iteration 1078/ 2384 | consumed samples: 34496 | elapsed time per iteration (ms): 226471.9 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.34% | FLOPs estimator: workload-aware | logical tokens/s: 74080.7 | valid tokens/s: 74080.7 | physical tokens/s: 74667.8 | loss tokens/s: 57658.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.981524E-01 | lm-global-supervised-tokens: 1.305812E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 2.510434E-01 | lm-token-buffer-mib: 2.332577E+02 | global_load_balancing_loss: 9.997511E-01 | loss scale: 1.0 | grad norm: 0.398 | params norm: 11183.779 | params rms: 0.060557 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:09:32.242532 | finish at 2026-10-17 19:49:13 + [2026-10-14 09:43:20] iteration 1079/ 2384 | consumed samples: 34528 | elapsed time per iteration (ms): 219465.5 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.25% | FLOPs estimator: workload-aware | logical tokens/s: 76445.6 | valid tokens/s: 76445.6 | physical tokens/s: 77297.6 | loss tokens/s: 59495.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.524089E-01 | lm-global-supervised-tokens: 1.305722E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.697651E-02 | lm-token-buffer-mib: 2.338020E+02 | global_load_balancing_loss: 9.998580E-01 | loss scale: 1.0 | grad norm: 0.410 | params norm: 11183.744 | params rms: 0.060556 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:33:22.444983 | finish at 2026-10-17 17:16:43 + [2026-10-14 09:47:28] iteration 1080/ 2384 | consumed samples: 34560 | elapsed time per iteration (ms): 247782.1 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.78% | FLOPs estimator: workload-aware | logical tokens/s: 67709.4 | valid tokens/s: 67709.4 | physical tokens/s: 68489.3 | loss tokens/s: 54348.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.551488E-01 | lm-global-supervised-tokens: 1.346666E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.229120E-02 | lm-token-buffer-mib: 2.331079E+02 | global_load_balancing_loss: 9.998855E-01 | loss scale: 1.0 | grad norm: 0.115 | params norm: 11183.711 | params rms: 0.060556 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:45:07.883533 | finish at 2026-10-18 03:32:36 +times across ranks (ms): + save-checkpoint: + rank 0: 24306.90 + rank 1: 24306.90 + rank 2: 24306.94 + rank 3: 24306.91 + rank 4: 24306.90 + rank 5: 24306.92 + rank 6: 24306.89 + rank 7: 24306.90 + rank 8: 24307.10 + rank 9: 24307.09 + rank 10: 24307.11 + rank 11: 24307.07 + rank 12: 24307.08 + rank 13: 24307.06 + rank 14: 24307.06 + rank 15: 24307.09 + rank 16: 24306.01 + rank 17: 24305.91 + rank 18: 24305.97 + rank 19: 24305.92 + rank 20: 24305.91 + rank 21: 24305.93 + rank 22: 24305.92 + rank 23: 24305.93 + rank 24: 24305.15 + rank 25: 24305.13 + rank 26: 24305.14 + rank 27: 24305.15 + rank 28: 24305.11 + rank 29: 24305.11 + rank 30: 24305.11 + rank 31: 24305.10 + rank 32: 24307.39 + rank 33: 24307.38 + rank 34: 24307.35 + rank 35: 24307.37 + rank 36: 24307.33 + rank 37: 24307.41 + rank 38: 24307.35 + rank 39: 24307.34 + rank 40: 24307.09 + rank 41: 24307.05 + rank 42: 24307.08 + rank 43: 24307.08 + rank 44: 24307.03 + rank 45: 24307.04 + rank 46: 24307.03 + rank 47: 24307.02 + rank 48: 24309.29 + rank 49: 24309.33 + rank 50: 24309.32 + rank 51: 24309.29 + rank 52: 24309.30 + rank 53: 24309.30 + rank 54: 24309.29 + rank 55: 24309.29 + rank 56: 24307.28 + rank 57: 24307.33 + rank 58: 24307.28 + rank 59: 24307.28 + rank 60: 24307.29 + rank 61: 24307.26 + rank 62: 24307.25 + rank 63: 24307.25 + [2026-10-14 09:51:32] iteration 1081/ 2384 | consumed samples: 34592 | elapsed time per iteration (ms): 220142.5 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.05% | FLOPs estimator: workload-aware | logical tokens/s: 76210.6 | valid tokens/s: 76210.6 | physical tokens/s: 76927.3 | loss tokens/s: 57008.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.051285E-01 | lm-global-supervised-tokens: 1.255004E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.898089E-02 | lm-token-buffer-mib: 2.354552E+02 | global_load_balancing_loss: 9.997454E-01 | loss scale: 1.0 | grad norm: 0.114 | params norm: 11183.677 | params rms: 0.060556 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:40:45.664974 | finish at 2026-10-17 17:32:18 + [2026-10-14 09:55:50] iteration 1082/ 2384 | consumed samples: 34624 | elapsed time per iteration (ms): 257653.5 | throughput per GPU (TFLOP/s/GPU): 16.9 | MFU 5.42% | FLOPs estimator: workload-aware | logical tokens/s: 65115.3 | valid tokens/s: 65115.3 | physical tokens/s: 65542.9 | loss tokens/s: 52227.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.117937E-01 | lm-global-supervised-tokens: 1.345651E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.300512E-02 | lm-token-buffer-mib: 2.341978E+02 | global_load_balancing_loss: 9.999726E-01 | loss scale: 1.0 | grad norm: 0.179 | params norm: 11183.640 | params rms: 0.060556 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:11:04.914222 | finish at 2026-10-18 07:06:55 + [2026-10-14 09:59:30] iteration 1083/ 2384 | consumed samples: 34656 | elapsed time per iteration (ms): 220038.0 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.28% | FLOPs estimator: workload-aware | logical tokens/s: 76246.8 | valid tokens/s: 76246.8 | physical tokens/s: 76657.6 | loss tokens/s: 53564.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.397920E-01 | lm-global-supervised-tokens: 1.178612E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.383863E-02 | lm-token-buffer-mib: 2.329685E+02 | global_load_balancing_loss: 1.000994E+00 | loss scale: 1.0 | grad norm: 0.157 | params norm: 11183.605 | params rms: 0.060556 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:31:09.474741 | finish at 2026-10-17 17:30:40 + [2026-10-14 10:03:15] iteration 1084/ 2384 | consumed samples: 34688 | elapsed time per iteration (ms): 224568.4 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.16% | FLOPs estimator: workload-aware | logical tokens/s: 74708.7 | valid tokens/s: 74708.6 | physical tokens/s: 75961.8 | loss tokens/s: 60992.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.445767E-01 | lm-global-supervised-tokens: 1.369687E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.081222E-02 | lm-token-buffer-mib: 2.356907E+02 | global_load_balancing_loss: 9.997973E-01 | loss scale: 1.0 | grad norm: 0.253 | params norm: 11183.574 | params rms: 0.060555 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:05:38.903141 | finish at 2026-10-17 19:08:54 + [2026-10-14 10:06:58] iteration 1085/ 2384 | consumed samples: 34720 | elapsed time per iteration (ms): 223295.8 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.74% | FLOPs estimator: workload-aware | logical tokens/s: 75134.5 | valid tokens/s: 75134.5 | physical tokens/s: 75934.4 | loss tokens/s: 56003.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.350496E-01 | lm-global-supervised-tokens: 1.250537E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.163528E-02 | lm-token-buffer-mib: 2.356187E+02 | global_load_balancing_loss: 1.000508E+00 | loss scale: 1.0 | grad norm: 0.303 | params norm: 11183.541 | params rms: 0.060555 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:34:21.186247 | finish at 2026-10-17 18:41:19 + [2026-10-14 10:10:40] iteration 1086/ 2384 | consumed samples: 34752 | elapsed time per iteration (ms): 222347.0 | throughput per GPU (TFLOP/s/GPU): 24.8 | MFU 7.96% | FLOPs estimator: workload-aware | logical tokens/s: 75455.0 | valid tokens/s: 75455.0 | physical tokens/s: 75948.7 | loss tokens/s: 56357.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.826113E-01 | lm-global-supervised-tokens: 1.253088E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.499984E-02 | lm-token-buffer-mib: 2.342728E+02 | global_load_balancing_loss: 1.001277E+00 | loss scale: 1.0 | grad norm: 0.357 | params norm: 11183.504 | params rms: 0.060555 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:10:06.406158 | finish at 2026-10-17 18:20:47 + [2026-10-14 10:14:21] iteration 1087/ 2384 | consumed samples: 34784 | elapsed time per iteration (ms): 220876.8 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.82% | FLOPs estimator: workload-aware | logical tokens/s: 75957.3 | valid tokens/s: 75957.3 | physical tokens/s: 76630.7 | loss tokens/s: 54147.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.458504E-01 | lm-global-supervised-tokens: 1.195994E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.324779E-02 | lm-token-buffer-mib: 2.340757E+02 | global_load_balancing_loss: 9.989264E-01 | loss scale: 1.0 | grad norm: 0.145 | params norm: 11183.466 | params rms: 0.060555 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:34:37.218646 | finish at 2026-10-17 17:48:58 + [2026-10-14 10:18:02] iteration 1088/ 2384 | consumed samples: 34816 | elapsed time per iteration (ms): 220960.4 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.59% | FLOPs estimator: workload-aware | logical tokens/s: 75928.6 | valid tokens/s: 75928.6 | physical tokens/s: 76615.1 | loss tokens/s: 57948.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.228563E-01 | lm-global-supervised-tokens: 1.280434E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.501208E-02 | lm-token-buffer-mib: 2.335543E+02 | global_load_balancing_loss: 9.996323E-01 | loss scale: 1.0 | grad norm: 0.115 | params norm: 11183.428 | params rms: 0.060555 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:32:44.713142 | finish at 2026-10-17 17:50:47 + [2026-10-14 10:21:48] iteration 1089/ 2384 | consumed samples: 34848 | elapsed time per iteration (ms): 226032.0 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.41% | FLOPs estimator: workload-aware | logical tokens/s: 74224.9 | valid tokens/s: 74224.9 | physical tokens/s: 75278.2 | loss tokens/s: 59536.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.811814E-01 | lm-global-supervised-tokens: 1.345711E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.354709E-02 | lm-token-buffer-mib: 2.371779E+02 | global_load_balancing_loss: 1.002993E+00 | loss scale: 1.0 | grad norm: 0.143 | params norm: 11183.392 | params rms: 0.060554 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:18:31.498129 | finish at 2026-10-17 19:40:20 + [2026-10-14 10:25:57] iteration 1090/ 2384 | consumed samples: 34880 | elapsed time per iteration (ms): 248354.4 | throughput per GPU (TFLOP/s/GPU): 16.9 | MFU 5.43% | FLOPs estimator: workload-aware | logical tokens/s: 67553.4 | valid tokens/s: 67553.4 | physical tokens/s: 68538.7 | loss tokens/s: 53927.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.921599E-01 | lm-global-supervised-tokens: 1.339324E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.298130E-02 | lm-token-buffer-mib: 2.349838E+02 | global_load_balancing_loss: 1.000211E+00 | loss scale: 1.0 | grad norm: 0.157 | params norm: 11183.353 | params rms: 0.060554 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:16:10.651496 | finish at 2026-10-18 03:42:07 + [2026-10-14 10:29:40] iteration 1091/ 2384 | consumed samples: 34912 | elapsed time per iteration (ms): 223913.0 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.72% | FLOPs estimator: workload-aware | logical tokens/s: 74927.4 | valid tokens/s: 74927.4 | physical tokens/s: 75573.6 | loss tokens/s: 56292.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.268439E-01 | lm-global-supervised-tokens: 1.260452E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.811391E-02 | lm-token-buffer-mib: 2.347924E+02 | global_load_balancing_loss: 1.001559E+00 | loss scale: 1.0 | grad norm: 0.165 | params norm: 11183.313 | params rms: 0.060554 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:25:19.493416 | finish at 2026-10-17 18:55:00 + [2026-10-14 10:33:24] iteration 1092/ 2384 | consumed samples: 34944 | elapsed time per iteration (ms): 223326.3 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.38% | FLOPs estimator: workload-aware | logical tokens/s: 75124.0 | valid tokens/s: 75124.0 | physical tokens/s: 75958.5 | loss tokens/s: 60976.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.845236E-01 | lm-global-supervised-tokens: 1.361765E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.427085E-02 | lm-token-buffer-mib: 2.349209E+02 | global_load_balancing_loss: 1.000414E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11183.276 | params rms: 0.060554 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:08:57.518487 | finish at 2026-10-17 18:42:21 + [2026-10-14 10:37:09] iteration 1093/ 2384 | consumed samples: 34976 | elapsed time per iteration (ms): 224877.6 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.71% | FLOPs estimator: workload-aware | logical tokens/s: 74605.9 | valid tokens/s: 74605.9 | physical tokens/s: 75214.5 | loss tokens/s: 59875.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.181280E-01 | lm-global-supervised-tokens: 1.346465E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.426322E-02 | lm-token-buffer-mib: 2.344702E+02 | global_load_balancing_loss: 9.991611E-01 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11183.241 | params rms: 0.060554 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:38:36.923675 | finish at 2026-10-17 19:15:46 + [2026-10-14 10:40:49] iteration 1094/ 2384 | consumed samples: 35008 | elapsed time per iteration (ms): 220321.5 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.39% | FLOPs estimator: workload-aware | logical tokens/s: 76148.6 | valid tokens/s: 76148.6 | physical tokens/s: 76957.8 | loss tokens/s: 56342.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.810268E-01 | lm-global-supervised-tokens: 1.241356E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.774087E-02 | lm-token-buffer-mib: 2.349993E+02 | global_load_balancing_loss: 1.000816E+00 | loss scale: 1.0 | grad norm: 0.163 | params norm: 11183.205 | params rms: 0.060553 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:56:54.763377 | finish at 2026-10-17 17:37:44 + [2026-10-14 10:44:32] iteration 1095/ 2384 | consumed samples: 35040 | elapsed time per iteration (ms): 222559.5 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.14% | FLOPs estimator: workload-aware | logical tokens/s: 75383.0 | valid tokens/s: 75383.0 | physical tokens/s: 76196.6 | loss tokens/s: 53294.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.213227E-01 | lm-global-supervised-tokens: 1.186126E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.633684E-02 | lm-token-buffer-mib: 2.333279E+02 | global_load_balancing_loss: 1.000585E+00 | loss scale: 1.0 | grad norm: 0.102 | params norm: 11183.167 | params rms: 0.060553 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:41:19.163204 | finish at 2026-10-17 18:25:51 + [2026-10-14 10:48:18] iteration 1096/ 2384 | consumed samples: 35072 | elapsed time per iteration (ms): 226830.9 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.35% | FLOPs estimator: workload-aware | logical tokens/s: 73963.5 | valid tokens/s: 73963.5 | physical tokens/s: 75818.1 | loss tokens/s: 57064.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.757113E-01 | lm-global-supervised-tokens: 1.294406E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.464059E-02 | lm-token-buffer-mib: 2.369113E+02 | global_load_balancing_loss: 1.000995E+00 | loss scale: 1.0 | grad norm: 0.131 | params norm: 11183.129 | params rms: 0.060553 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:09:18.173346 | finish at 2026-10-17 19:57:37 + [2026-10-14 10:52:02] iteration 1097/ 2384 | consumed samples: 35104 | elapsed time per iteration (ms): 224121.6 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.15% | FLOPs estimator: workload-aware | logical tokens/s: 74857.5 | valid tokens/s: 74857.5 | physical tokens/s: 75609.5 | loss tokens/s: 55444.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.182475E-01 | lm-global-supervised-tokens: 1.242625E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.290655E-02 | lm-token-buffer-mib: 2.334958E+02 | global_load_balancing_loss: 9.992878E-01 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11183.093 | params rms: 0.060553 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:07:24.556774 | finish at 2026-10-17 18:59:27 + [2026-10-14 10:55:46] iteration 1098/ 2384 | consumed samples: 35136 | elapsed time per iteration (ms): 223606.2 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.05% | FLOPs estimator: workload-aware | logical tokens/s: 75030.1 | valid tokens/s: 75030.1 | physical tokens/s: 76063.1 | loss tokens/s: 57618.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.201472E-01 | lm-global-supervised-tokens: 1.288378E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.251981E-02 | lm-token-buffer-mib: 2.344962E+02 | global_load_balancing_loss: 9.999809E-01 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11183.059 | params rms: 0.060553 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:52:37.598622 | finish at 2026-10-17 18:48:24 + [2026-10-14 10:59:27] iteration 1099/ 2384 | consumed samples: 35168 | elapsed time per iteration (ms): 220766.6 | throughput per GPU (TFLOP/s/GPU): 23.4 | MFU 7.51% | FLOPs estimator: workload-aware | logical tokens/s: 75995.2 | valid tokens/s: 75995.2 | physical tokens/s: 76522.3 | loss tokens/s: 57378.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.213689E-01 | lm-global-supervised-tokens: 1.266730E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.100772E-02 | lm-token-buffer-mib: 2.334371E+02 | global_load_balancing_loss: 1.000521E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11183.022 | params rms: 0.060552 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:48:05.094343 | finish at 2026-10-17 17:47:32 + [2026-10-14 11:03:31] iteration 1100/ 2384 | consumed samples: 35200 | elapsed time per iteration (ms): 244373.3 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 7.00% | FLOPs estimator: workload-aware | logical tokens/s: 68653.9 | valid tokens/s: 68653.9 | physical tokens/s: 69313.8 | loss tokens/s: 53044.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.579645E-01 | lm-global-supervised-tokens: 1.296263E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.196061E-02 | lm-token-buffer-mib: 2.339234E+02 | global_load_balancing_loss: 1.000565E+00 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11182.988 | params rms: 0.060552 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:09:35.293113 | finish at 2026-10-18 02:13:07 + [2026-10-14 11:07:15] iteration 1101/ 2384 | consumed samples: 35232 | elapsed time per iteration (ms): 223406.2 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.40% | FLOPs estimator: workload-aware | logical tokens/s: 75097.2 | valid tokens/s: 75097.2 | physical tokens/s: 76199.1 | loss tokens/s: 59573.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.904733E-01 | lm-global-supervised-tokens: 1.330917E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.383543E-02 | lm-token-buffer-mib: 2.359150E+02 | global_load_balancing_loss: 9.991226E-01 | loss scale: 1.0 | grad norm: 0.104 | params norm: 11182.954 | params rms: 0.060552 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:37:10.148395 | finish at 2026-10-17 18:44:25 + [2026-10-14 11:10:56] iteration 1102/ 2384 | consumed samples: 35264 | elapsed time per iteration (ms): 221838.9 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.03% | FLOPs estimator: workload-aware | logical tokens/s: 75627.9 | valid tokens/s: 75627.9 | physical tokens/s: 76257.1 | loss tokens/s: 55831.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.429687E-01 | lm-global-supervised-tokens: 1.238559E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.929663E-02 | lm-token-buffer-mib: 2.333137E+02 | global_load_balancing_loss: 1.001863E+00 | loss scale: 1.0 | grad norm: 0.129 | params norm: 11182.921 | params rms: 0.060552 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:59:57.483363 | finish at 2026-10-17 18:10:54 + [2026-10-14 11:14:37] iteration 1103/ 2384 | consumed samples: 35296 | elapsed time per iteration (ms): 220858.9 | throughput per GPU (TFLOP/s/GPU): 24.5 | MFU 7.85% | FLOPs estimator: workload-aware | logical tokens/s: 75963.3 | valid tokens/s: 75963.3 | physical tokens/s: 76490.9 | loss tokens/s: 55666.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.674177E-01 | lm-global-supervised-tokens: 1.229451E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.170615E-02 | lm-token-buffer-mib: 2.346090E+02 | global_load_balancing_loss: 1.001233E+00 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11182.885 | params rms: 0.060552 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:35:20.260763 | finish at 2026-10-17 17:49:58 + [2026-10-14 11:18:21] iteration 1104/ 2384 | consumed samples: 35328 | elapsed time per iteration (ms): 223366.4 | throughput per GPU (TFLOP/s/GPU): 24.9 | MFU 8.00% | FLOPs estimator: workload-aware | logical tokens/s: 75110.4 | valid tokens/s: 75110.4 | physical tokens/s: 75662.0 | loss tokens/s: 54272.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.353112E-01 | lm-global-supervised-tokens: 1.212268E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.771377E-02 | lm-token-buffer-mib: 2.344925E+02 | global_load_balancing_loss: 1.001310E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11182.845 | params rms: 0.060551 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:25:08.955383 | finish at 2026-10-17 18:43:30 + [2026-10-14 11:22:03] iteration 1105/ 2384 | consumed samples: 35360 | elapsed time per iteration (ms): 222317.9 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.75% | FLOPs estimator: workload-aware | logical tokens/s: 75464.9 | valid tokens/s: 75464.9 | physical tokens/s: 75895.9 | loss tokens/s: 59864.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.259885E-01 | lm-global-supervised-tokens: 1.330884E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.263834E-02 | lm-token-buffer-mib: 2.328177E+02 | global_load_balancing_loss: 9.999495E-01 | loss scale: 1.0 | grad norm: 0.137 | params norm: 11182.804 | params rms: 0.060551 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:59:04.648000 | finish at 2026-10-17 18:21:08 + [2026-10-14 11:25:49] iteration 1106/ 2384 | consumed samples: 35392 | elapsed time per iteration (ms): 226455.0 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.06% | FLOPs estimator: workload-aware | logical tokens/s: 74086.1 | valid tokens/s: 74086.1 | physical tokens/s: 74913.3 | loss tokens/s: 53823.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.162552E-01 | lm-global-supervised-tokens: 1.218851E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.068851E-02 | lm-token-buffer-mib: 2.358022E+02 | global_load_balancing_loss: 1.000914E+00 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11182.763 | params rms: 0.060551 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:23:29.440541 | finish at 2026-10-17 19:49:19 + [2026-10-14 11:29:32] iteration 1107/ 2384 | consumed samples: 35424 | elapsed time per iteration (ms): 222624.9 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.36% | FLOPs estimator: workload-aware | logical tokens/s: 75360.8 | valid tokens/s: 75360.8 | physical tokens/s: 76708.6 | loss tokens/s: 59413.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.855042E-01 | lm-global-supervised-tokens: 1.322683E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.354562E-02 | lm-token-buffer-mib: 2.336096E+02 | global_load_balancing_loss: 1.000378E+00 | loss scale: 1.0 | grad norm: 0.406 | params norm: 11182.726 | params rms: 0.060551 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:58:12.026355 | finish at 2026-10-17 18:27:44 + [2026-10-14 11:33:13] iteration 1108/ 2384 | consumed samples: 35456 | elapsed time per iteration (ms): 220735.5 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.74% | FLOPs estimator: workload-aware | logical tokens/s: 76005.9 | valid tokens/s: 76005.9 | physical tokens/s: 76706.2 | loss tokens/s: 56090.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.455269E-01 | lm-global-supervised-tokens: 1.238108E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.308458E-02 | lm-token-buffer-mib: 2.322646E+02 | global_load_balancing_loss: 1.000432E+00 | loss scale: 1.0 | grad norm: 0.340 | params norm: 11182.690 | params rms: 0.060551 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:14:18.537369 | finish at 2026-10-17 17:47:31 + [2026-10-14 11:36:57] iteration 1109/ 2384 | consumed samples: 35488 | elapsed time per iteration (ms): 224333.7 | throughput per GPU (TFLOP/s/GPU): 23.8 | MFU 7.62% | FLOPs estimator: workload-aware | logical tokens/s: 74786.7 | valid tokens/s: 74786.7 | physical tokens/s: 75524.9 | loss tokens/s: 54963.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.096376E-01 | lm-global-supervised-tokens: 1.233018E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.617658E-02 | lm-token-buffer-mib: 2.350601E+02 | global_load_balancing_loss: 9.994410E-01 | loss scale: 1.0 | grad norm: 0.365 | params norm: 11182.653 | params rms: 0.060550 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:27:05.413316 | finish at 2026-10-17 19:04:03 + [2026-10-14 11:41:03] iteration 1110/ 2384 | consumed samples: 35520 | elapsed time per iteration (ms): 246216.2 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.55% | FLOPs estimator: workload-aware | logical tokens/s: 68140.0 | valid tokens/s: 68140.0 | physical tokens/s: 69113.6 | loss tokens/s: 54061.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.505387E-01 | lm-global-supervised-tokens: 1.331089E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.693429E-02 | lm-token-buffer-mib: 2.363725E+02 | global_load_balancing_loss: 9.998298E-01 | loss scale: 1.0 | grad norm: 0.176 | params norm: 11182.614 | params rms: 0.060550 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:07:59.436514 | finish at 2026-10-18 02:49:03 + [2026-10-14 11:44:48] iteration 1111/ 2384 | consumed samples: 35552 | elapsed time per iteration (ms): 224617.1 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.62% | FLOPs estimator: workload-aware | logical tokens/s: 74692.4 | valid tokens/s: 74692.3 | physical tokens/s: 75611.4 | loss tokens/s: 56868.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.769803E-01 | lm-global-supervised-tokens: 1.277366E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.393371E-02 | lm-token-buffer-mib: 2.343133E+02 | global_load_balancing_loss: 9.997798E-01 | loss scale: 1.0 | grad norm: 0.580 | params norm: 11182.573 | params rms: 0.060550 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:25:37.545537 | finish at 2026-10-17 19:10:26 + [2026-10-14 11:48:32] iteration 1112/ 2384 | consumed samples: 35584 | elapsed time per iteration (ms): 223836.1 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.38% | FLOPs estimator: workload-aware | logical tokens/s: 74952.9 | valid tokens/s: 74952.9 | physical tokens/s: 75882.1 | loss tokens/s: 55709.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.090871E-01 | lm-global-supervised-tokens: 1.246976E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.926113E-02 | lm-token-buffer-mib: 2.352331E+02 | global_load_balancing_loss: 1.000101E+00 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11182.532 | params rms: 0.060550 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:05:19.549353 | finish at 2026-10-17 18:53:51 + [2026-10-14 11:52:14] iteration 1113/ 2384 | consumed samples: 35616 | elapsed time per iteration (ms): 222591.5 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.23% | FLOPs estimator: workload-aware | logical tokens/s: 75372.0 | valid tokens/s: 75372.0 | physical tokens/s: 76391.5 | loss tokens/s: 58969.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.175312E-01 | lm-global-supervised-tokens: 1.312618E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.372049E-02 | lm-token-buffer-mib: 2.351252E+02 | global_load_balancing_loss: 1.003291E+00 | loss scale: 1.0 | grad norm: 0.662 | params norm: 11182.492 | params rms: 0.060550 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:35:13.808374 | finish at 2026-10-17 18:27:28 + [2026-10-14 11:55:57] iteration 1114/ 2384 | consumed samples: 35648 | elapsed time per iteration (ms): 222932.5 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.39% | FLOPs estimator: workload-aware | logical tokens/s: 75256.8 | valid tokens/s: 75256.8 | physical tokens/s: 76068.2 | loss tokens/s: 56696.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.400720E-01 | lm-global-supervised-tokens: 1.263959E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.341524E-02 | lm-token-buffer-mib: 2.342986E+02 | global_load_balancing_loss: 1.001170E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11182.451 | params rms: 0.060549 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:38:44.265771 | finish at 2026-10-17 18:34:42 + [2026-10-14 11:59:40] iteration 1115/ 2384 | consumed samples: 35680 | elapsed time per iteration (ms): 222211.1 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.41% | FLOPs estimator: workload-aware | logical tokens/s: 75501.1 | valid tokens/s: 75501.1 | physical tokens/s: 76459.8 | loss tokens/s: 57996.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.601749E-01 | lm-global-supervised-tokens: 1.288752E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.035921E-02 | lm-token-buffer-mib: 2.341079E+02 | global_load_balancing_loss: 1.000055E+00 | loss scale: 1.0 | grad norm: 0.119 | params norm: 11182.406 | params rms: 0.060549 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:19:45.915982 | finish at 2026-10-17 18:19:25 + [2026-10-14 12:03:21] iteration 1116/ 2384 | consumed samples: 35712 | elapsed time per iteration (ms): 220959.2 | throughput per GPU (TFLOP/s/GPU): 18.1 | MFU 5.81% | FLOPs estimator: workload-aware | logical tokens/s: 75929.0 | valid tokens/s: 75929.0 | physical tokens/s: 77208.7 | loss tokens/s: 57514.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.584048E-01 | lm-global-supervised-tokens: 1.270826E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.336531E-02 | lm-token-buffer-mib: 2.357101E+02 | global_load_balancing_loss: 9.996840E-01 | loss scale: 1.0 | grad norm: 0.474 | params norm: 11182.362 | params rms: 0.060549 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:49:36.229807 | finish at 2026-10-17 17:52:57 + [2026-10-14 12:06:59] iteration 1117/ 2384 | consumed samples: 35744 | elapsed time per iteration (ms): 218916.6 | throughput per GPU (TFLOP/s/GPU): 25.0 | MFU 8.01% | FLOPs estimator: workload-aware | logical tokens/s: 76637.3 | valid tokens/s: 76637.3 | physical tokens/s: 76760.3 | loss tokens/s: 53625.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.463943E-01 | lm-global-supervised-tokens: 1.173957E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.135894E-02 | lm-token-buffer-mib: 2.325615E+02 | global_load_balancing_loss: 9.998340E-01 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11182.320 | params rms: 0.060549 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:02:47.369341 | finish at 2026-10-17 17:09:47 + [2026-10-14 12:10:42] iteration 1118/ 2384 | consumed samples: 35776 | elapsed time per iteration (ms): 223027.8 | throughput per GPU (TFLOP/s/GPU): 24.6 | MFU 7.88% | FLOPs estimator: workload-aware | logical tokens/s: 75224.6 | valid tokens/s: 75224.6 | physical tokens/s: 75864.4 | loss tokens/s: 57624.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.677345E-01 | lm-global-supervised-tokens: 1.285190E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.441594E-02 | lm-token-buffer-mib: 2.337331E+02 | global_load_balancing_loss: 1.001207E+00 | loss scale: 1.0 | grad norm: 0.177 | params norm: 11182.282 | params rms: 0.060548 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:25:53.248632 | finish at 2026-10-17 18:36:36 + [2026-10-14 12:14:28] iteration 1119/ 2384 | consumed samples: 35808 | elapsed time per iteration (ms): 225513.4 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.78% | FLOPs estimator: workload-aware | logical tokens/s: 74395.6 | valid tokens/s: 74395.6 | physical tokens/s: 75063.4 | loss tokens/s: 54845.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.694006E-01 | lm-global-supervised-tokens: 1.236837E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.680098E-02 | lm-token-buffer-mib: 2.351963E+02 | global_load_balancing_loss: 9.990726E-01 | loss scale: 1.0 | grad norm: 0.194 | params norm: 11182.242 | params rms: 0.060548 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:14:34.443257 | finish at 2026-10-17 19:29:02 + [2026-10-14 12:18:34] iteration 1120/ 2384 | consumed samples: 35840 | elapsed time per iteration (ms): 245955.4 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.44% | FLOPs estimator: workload-aware | logical tokens/s: 68212.3 | valid tokens/s: 68212.3 | physical tokens/s: 68744.8 | loss tokens/s: 52032.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.190178E-01 | lm-global-supervised-tokens: 1.279776E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.627061E-02 | lm-token-buffer-mib: 2.335958E+02 | global_load_balancing_loss: 1.000460E+00 | loss scale: 1.0 | grad norm: 0.191 | params norm: 11182.198 | params rms: 0.060548 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:21:27.564110 | finish at 2026-10-18 02:40:02 +times across ranks (ms): + save-checkpoint: + rank 0: 24844.15 + rank 1: 24844.13 + rank 2: 24844.17 + rank 3: 24844.16 + rank 4: 24844.12 + rank 5: 24844.13 + rank 6: 24844.15 + rank 7: 24844.14 + rank 8: 24844.18 + rank 9: 24844.21 + rank 10: 24844.21 + rank 11: 24844.18 + rank 12: 24844.19 + rank 13: 24844.18 + rank 14: 24844.18 + rank 15: 24844.19 + rank 16: 24843.42 + rank 17: 24843.41 + rank 18: 24843.43 + rank 19: 24843.41 + rank 20: 24843.40 + rank 21: 24843.40 + rank 22: 24843.27 + rank 23: 24843.41 + rank 24: 24844.29 + rank 25: 24844.27 + rank 26: 24844.35 + rank 27: 24845.02 + rank 28: 24844.26 + rank 29: 24844.29 + rank 30: 24844.31 + rank 31: 24844.28 + rank 32: 24844.34 + rank 33: 24844.35 + rank 34: 24844.33 + rank 35: 24844.29 + rank 36: 24844.39 + rank 37: 24844.24 + rank 38: 24844.34 + rank 39: 24844.36 + rank 40: 24844.04 + rank 41: 24844.04 + rank 42: 24844.11 + rank 43: 24844.05 + rank 44: 24844.03 + rank 45: 24844.02 + rank 46: 24844.03 + rank 47: 24844.04 + rank 48: 24843.89 + rank 49: 24843.79 + rank 50: 24843.76 + rank 51: 24843.81 + rank 52: 24843.82 + rank 53: 24843.89 + rank 54: 24843.76 + rank 55: 24843.82 + rank 56: 24844.00 + rank 57: 24844.03 + rank 58: 24844.03 + rank 59: 24844.04 + rank 60: 24844.04 + rank 61: 24843.99 + rank 62: 24843.99 + rank 63: 24843.98 + [2026-10-14 12:22:46] iteration 1121/ 2384 | consumed samples: 35872 | elapsed time per iteration (ms): 227250.7 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.97% | FLOPs estimator: workload-aware | logical tokens/s: 73826.8 | valid tokens/s: 73826.8 | physical tokens/s: 74351.6 | loss tokens/s: 56061.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.793510E-01 | lm-global-supervised-tokens: 1.273995E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.616891E-02 | lm-token-buffer-mib: 2.341148E+02 | global_load_balancing_loss: 9.998733E-01 | loss scale: 1.0 | grad norm: 0.112 | params norm: 11182.152 | params rms: 0.060548 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:43:37.638613 | finish at 2026-10-17 20:06:24 + failed to log checkpoint /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/iter_0001120 as wandb artifact: Unable to connect to server to verify API token. + [2026-10-14 12:26:38] iteration 1122/ 2384 | consumed samples: 35904 | elapsed time per iteration (ms): 232294.0 | throughput per GPU (TFLOP/s/GPU): 20.2 | MFU 6.46% | FLOPs estimator: workload-aware | logical tokens/s: 72224.0 | valid tokens/s: 72224.0 | physical tokens/s: 72781.1 | loss tokens/s: 51704.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.128825E-01 | lm-global-supervised-tokens: 1.201058E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.072512E-02 | lm-token-buffer-mib: 2.327504E+02 | global_load_balancing_loss: 1.001018E+00 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11182.111 | params rms: 0.060547 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:25:55.046241 | finish at 2026-10-17 21:52:33 + [2026-10-14 12:30:21] iteration 1123/ 2384 | consumed samples: 35936 | elapsed time per iteration (ms): 222159.7 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.35% | FLOPs estimator: workload-aware | logical tokens/s: 75518.6 | valid tokens/s: 75518.6 | physical tokens/s: 76124.9 | loss tokens/s: 53748.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.813814E-01 | lm-global-supervised-tokens: 1.194077E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.601266E-02 | lm-token-buffer-mib: 2.323758E+02 | global_load_balancing_loss: 1.001514E+00 | loss scale: 1.0 | grad norm: 0.239 | params norm: 11182.072 | params rms: 0.060547 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:49:03.328982 | finish at 2026-10-17 18:19:24 + [2026-10-14 12:34:00] iteration 1124/ 2384 | consumed samples: 35968 | elapsed time per iteration (ms): 219044.1 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.96% | FLOPs estimator: workload-aware | logical tokens/s: 76592.8 | valid tokens/s: 76592.8 | physical tokens/s: 76851.2 | loss tokens/s: 55658.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.692767E-01 | lm-global-supervised-tokens: 1.219168E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.513077E-02 | lm-token-buffer-mib: 2.328722E+02 | global_load_balancing_loss: 9.998939E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11182.031 | params rms: 0.060547 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:39:55.509882 | finish at 2026-10-17 17:13:55 + [2026-10-14 12:37:42] iteration 1125/ 2384 | consumed samples: 36000 | elapsed time per iteration (ms): 222690.0 | throughput per GPU (TFLOP/s/GPU): 27.8 | MFU 8.90% | FLOPs estimator: workload-aware | logical tokens/s: 75338.8 | valid tokens/s: 75338.8 | physical tokens/s: 75699.6 | loss tokens/s: 54627.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.337592E-01 | lm-global-supervised-tokens: 1.216495E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.268941E-02 | lm-token-buffer-mib: 2.338721E+02 | global_load_balancing_loss: 1.000721E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11181.987 | params rms: 0.060547 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:52:46.650038 | finish at 2026-10-17 18:30:29 + [2026-10-14 12:41:22] iteration 1126/ 2384 | consumed samples: 36032 | elapsed time per iteration (ms): 219827.0 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.92% | FLOPs estimator: workload-aware | logical tokens/s: 76320.0 | valid tokens/s: 76320.0 | physical tokens/s: 77024.1 | loss tokens/s: 58701.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.230751E-01 | lm-global-supervised-tokens: 1.290411E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.217936E-02 | lm-token-buffer-mib: 2.327385E+02 | global_load_balancing_loss: 1.000338E+00 | loss scale: 1.0 | grad norm: 0.115 | params norm: 11181.945 | params rms: 0.060547 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:49:02.349681 | finish at 2026-10-17 17:30:24 + [2026-10-14 12:45:04] iteration 1127/ 2384 | consumed samples: 36064 | elapsed time per iteration (ms): 221615.7 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.43% | FLOPs estimator: workload-aware | logical tokens/s: 75703.9 | valid tokens/s: 75703.9 | physical tokens/s: 76482.7 | loss tokens/s: 58279.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.379575E-01 | lm-global-supervised-tokens: 1.291564E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.233308E-02 | lm-token-buffer-mib: 2.338021E+02 | global_load_balancing_loss: 9.996375E-01 | loss scale: 1.0 | grad norm: 0.188 | params norm: 11181.904 | params rms: 0.060546 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:22:50.963379 | finish at 2026-10-17 18:07:55 + [2026-10-14 12:48:48] iteration 1128/ 2384 | consumed samples: 36096 | elapsed time per iteration (ms): 223992.8 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.54% | FLOPs estimator: workload-aware | logical tokens/s: 74900.6 | valid tokens/s: 74900.6 | physical tokens/s: 76239.0 | loss tokens/s: 59866.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.220625E-01 | lm-global-supervised-tokens: 1.340966E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.927867E-02 | lm-token-buffer-mib: 2.360912E+02 | global_load_balancing_loss: 1.000002E+00 | loss scale: 1.0 | grad norm: 0.095 | params norm: 11181.862 | params rms: 0.060546 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:08:54.952904 | finish at 2026-10-17 18:57:43 + [2026-10-14 12:52:33] iteration 1129/ 2384 | consumed samples: 36128 | elapsed time per iteration (ms): 225786.9 | throughput per GPU (TFLOP/s/GPU): 26.4 | MFU 8.46% | FLOPs estimator: workload-aware | logical tokens/s: 74305.4 | valid tokens/s: 74305.4 | physical tokens/s: 74691.0 | loss tokens/s: 55320.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.181727E-01 | lm-global-supervised-tokens: 1.249054E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.178050E-02 | lm-token-buffer-mib: 2.337037E+02 | global_load_balancing_loss: 1.000409E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11181.822 | params rms: 0.060546 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:42:42.511381 | finish at 2026-10-17 19:35:16 + [2026-10-14 12:56:35] iteration 1130/ 2384 | consumed samples: 36160 | elapsed time per iteration (ms): 241118.9 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.50% | FLOPs estimator: workload-aware | logical tokens/s: 69580.5 | valid tokens/s: 69580.5 | physical tokens/s: 70010.4 | loss tokens/s: 52707.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.024208E-01 | lm-global-supervised-tokens: 1.270877E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.213890E-02 | lm-token-buffer-mib: 2.332341E+02 | global_load_balancing_loss: 1.000672E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11181.780 | params rms: 0.060546 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:59:23.112336 | finish at 2026-10-18 00:55:58 + [2026-10-14 13:00:16] iteration 1131/ 2384 | consumed samples: 36192 | elapsed time per iteration (ms): 221599.7 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.90% | FLOPs estimator: workload-aware | logical tokens/s: 75709.5 | valid tokens/s: 75709.5 | physical tokens/s: 76335.1 | loss tokens/s: 58007.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.081181E-01 | lm-global-supervised-tokens: 1.285448E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.213198E-02 | lm-token-buffer-mib: 2.328573E+02 | global_load_balancing_loss: 1.000414E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11181.737 | params rms: 0.060545 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:07:44.486504 | finish at 2026-10-17 18:08:01 + [2026-10-14 13:03:59] iteration 1132/ 2384 | consumed samples: 36224 | elapsed time per iteration (ms): 223055.8 | throughput per GPU (TFLOP/s/GPU): 19.5 | MFU 6.25% | FLOPs estimator: workload-aware | logical tokens/s: 75215.2 | valid tokens/s: 75215.2 | physical tokens/s: 75934.9 | loss tokens/s: 58496.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.480582E-01 | lm-global-supervised-tokens: 1.304790E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.636865E-02 | lm-token-buffer-mib: 2.340242E+02 | global_load_balancing_loss: 9.999846E-01 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11181.696 | params rms: 0.060545 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:34:25.866029 | finish at 2026-10-17 18:38:25 + [2026-10-14 13:07:41] iteration 1133/ 2384 | consumed samples: 36256 | elapsed time per iteration (ms): 222097.6 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.12% | FLOPs estimator: workload-aware | logical tokens/s: 75539.7 | valid tokens/s: 75539.7 | physical tokens/s: 75962.6 | loss tokens/s: 54642.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.312254E-01 | lm-global-supervised-tokens: 1.213587E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.312799E-02 | lm-token-buffer-mib: 2.326147E+02 | global_load_balancing_loss: 1.000707E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11181.656 | params rms: 0.060545 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:10:44.082666 | finish at 2026-10-17 18:18:25 + [2026-10-14 13:11:23] iteration 1134/ 2384 | consumed samples: 36288 | elapsed time per iteration (ms): 221530.8 | throughput per GPU (TFLOP/s/GPU): 26.0 | MFU 8.33% | FLOPs estimator: workload-aware | logical tokens/s: 75733.0 | valid tokens/s: 75733.0 | physical tokens/s: 76028.7 | loss tokens/s: 51773.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.950815E-01 | lm-global-supervised-tokens: 1.146952E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.752122E-02 | lm-token-buffer-mib: 2.325392E+02 | global_load_balancing_loss: 1.000108E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11181.616 | params rms: 0.060545 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:55:13.439631 | finish at 2026-10-17 18:06:36 + [2026-10-14 13:15:11] iteration 1135/ 2384 | consumed samples: 36320 | elapsed time per iteration (ms): 228600.6 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.06% | FLOPs estimator: workload-aware | logical tokens/s: 73390.9 | valid tokens/s: 73390.9 | physical tokens/s: 74699.2 | loss tokens/s: 56377.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.071818E-01 | lm-global-supervised-tokens: 1.288795E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.337990E-02 | lm-token-buffer-mib: 2.381560E+02 | global_load_balancing_loss: 1.001302E+00 | loss scale: 1.0 | grad norm: 0.135 | params norm: 11181.575 | params rms: 0.060545 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:18:42.087466 | finish at 2026-10-17 20:33:54 + [2026-10-14 13:18:57] iteration 1136/ 2384 | consumed samples: 36352 | elapsed time per iteration (ms): 225445.6 | throughput per GPU (TFLOP/s/GPU): 23.5 | MFU 7.53% | FLOPs estimator: workload-aware | logical tokens/s: 74418.0 | valid tokens/s: 74418.0 | physical tokens/s: 75298.9 | loss tokens/s: 54116.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.754291E-01 | lm-global-supervised-tokens: 1.220020E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.574330E-02 | lm-token-buffer-mib: 2.362558E+02 | global_load_balancing_loss: 9.999603E-01 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11181.537 | params rms: 0.060544 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:09:16.063316 | finish at 2026-10-17 19:28:13 + [2026-10-14 13:22:40] iteration 1137/ 2384 | consumed samples: 36384 | elapsed time per iteration (ms): 223173.6 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.93% | FLOPs estimator: workload-aware | logical tokens/s: 75175.6 | valid tokens/s: 75175.6 | physical tokens/s: 75596.0 | loss tokens/s: 56049.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.563683E-01 | lm-global-supervised-tokens: 1.250872E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.947340E-02 | lm-token-buffer-mib: 2.336648E+02 | global_load_balancing_loss: 1.001735E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11181.502 | params rms: 0.060544 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:18:17.526420 | finish at 2026-10-17 18:40:58 + [2026-10-14 13:26:20] iteration 1138/ 2384 | consumed samples: 36416 | elapsed time per iteration (ms): 220043.8 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.41% | FLOPs estimator: workload-aware | logical tokens/s: 76244.8 | valid tokens/s: 76244.8 | physical tokens/s: 76858.0 | loss tokens/s: 57938.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.169900E-01 | lm-global-supervised-tokens: 1.274892E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.348952E-02 | lm-token-buffer-mib: 2.337864E+02 | global_load_balancing_loss: 1.001195E+00 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11181.468 | params rms: 0.060544 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:09:34.522363 | finish at 2026-10-17 17:35:55 + [2026-10-14 13:30:01] iteration 1139/ 2384 | consumed samples: 36448 | elapsed time per iteration (ms): 221045.0 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.01% | FLOPs estimator: workload-aware | logical tokens/s: 75899.5 | valid tokens/s: 75899.5 | physical tokens/s: 76528.4 | loss tokens/s: 57215.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.666632E-01 | lm-global-supervised-tokens: 1.264730E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.232658E-02 | lm-token-buffer-mib: 2.340295E+02 | global_load_balancing_loss: 1.000953E+00 | loss scale: 1.0 | grad norm: 0.148 | params norm: 11181.433 | params rms: 0.060544 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:26:41.024204 | finish at 2026-10-17 17:56:42 + [2026-10-14 13:34:03] iteration 1140/ 2384 | consumed samples: 36480 | elapsed time per iteration (ms): 242299.1 | throughput per GPU (TFLOP/s/GPU): 17.8 | MFU 5.70% | FLOPs estimator: workload-aware | logical tokens/s: 69241.8 | valid tokens/s: 69241.7 | physical tokens/s: 70348.5 | loss tokens/s: 55403.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.606672E-01 | lm-global-supervised-tokens: 1.342419E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.914183E-02 | lm-token-buffer-mib: 2.359097E+02 | global_load_balancing_loss: 9.994892E-01 | loss scale: 1.0 | grad norm: 0.118 | params norm: 11181.395 | params rms: 0.060544 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:43:40.102251 | finish at 2026-10-18 01:17:44 + [2026-10-14 13:37:44] iteration 1141/ 2384 | consumed samples: 36512 | elapsed time per iteration (ms): 220925.5 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.97% | FLOPs estimator: workload-aware | logical tokens/s: 75940.5 | valid tokens/s: 75940.5 | physical tokens/s: 76814.6 | loss tokens/s: 57240.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.006564E-01 | lm-global-supervised-tokens: 1.264585E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.358636E-02 | lm-token-buffer-mib: 2.338711E+02 | global_load_balancing_loss: 1.000044E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11181.365 | params rms: 0.060543 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:16:50.458630 | finish at 2026-10-17 17:54:35 + [2026-10-14 13:41:33] iteration 1142/ 2384 | consumed samples: 36544 | elapsed time per iteration (ms): 228916.4 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.23% | FLOPs estimator: workload-aware | logical tokens/s: 73289.5 | valid tokens/s: 73289.5 | physical tokens/s: 74581.9 | loss tokens/s: 55213.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.172254E-01 | lm-global-supervised-tokens: 1.263918E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.297668E-02 | lm-token-buffer-mib: 2.371159E+02 | global_load_balancing_loss: 9.989482E-01 | loss scale: 1.0 | grad norm: 0.125 | params norm: 11181.333 | params rms: 0.060543 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:58:34.199541 | finish at 2026-10-17 20:40:08 + [2026-10-14 13:45:20] iteration 1143/ 2384 | consumed samples: 36576 | elapsed time per iteration (ms): 226191.6 | throughput per GPU (TFLOP/s/GPU): 19.4 | MFU 6.23% | FLOPs estimator: workload-aware | logical tokens/s: 74172.5 | valid tokens/s: 74172.5 | physical tokens/s: 75155.5 | loss tokens/s: 59791.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.935408E-01 | lm-global-supervised-tokens: 1.352424E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.633905E-02 | lm-token-buffer-mib: 2.346249E+02 | global_load_balancing_loss: 9.996061E-01 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11181.300 | params rms: 0.060543 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:58:23.758248 | finish at 2026-10-17 19:43:43 + [2026-10-14 13:49:06] iteration 1144/ 2384 | consumed samples: 36608 | elapsed time per iteration (ms): 226470.9 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.58% | FLOPs estimator: workload-aware | logical tokens/s: 74081.0 | valid tokens/s: 74081.0 | physical tokens/s: 75541.6 | loss tokens/s: 56145.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.321909E-01 | lm-global-supervised-tokens: 1.271520E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.262917E-02 | lm-token-buffer-mib: 2.357102E+02 | global_load_balancing_loss: 1.000461E+00 | loss scale: 1.0 | grad norm: 0.148 | params norm: 11181.264 | params rms: 0.060543 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:00:23.858128 | finish at 2026-10-17 19:49:30 + [2026-10-14 13:52:51] iteration 1145/ 2384 | consumed samples: 36640 | elapsed time per iteration (ms): 224867.8 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.94% | FLOPs estimator: workload-aware | logical tokens/s: 74609.2 | valid tokens/s: 74609.2 | physical tokens/s: 76187.7 | loss tokens/s: 55309.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.116278E-01 | lm-global-supervised-tokens: 1.243741E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.102012E-02 | lm-token-buffer-mib: 2.355402E+02 | global_load_balancing_loss: 1.001287E+00 | loss scale: 1.0 | grad norm: 0.147 | params norm: 11181.229 | params rms: 0.060543 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:23:31.192971 | finish at 2026-10-17 19:16:22 + [2026-10-14 13:56:34] iteration 1146/ 2384 | consumed samples: 36672 | elapsed time per iteration (ms): 223238.8 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.08% | FLOPs estimator: workload-aware | logical tokens/s: 75153.7 | valid tokens/s: 75153.7 | physical tokens/s: 75850.9 | loss tokens/s: 58966.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.787140E-01 | lm-global-supervised-tokens: 1.316355E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.344360E-02 | lm-token-buffer-mib: 2.346933E+02 | global_load_balancing_loss: 1.002292E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11181.195 | params rms: 0.060543 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:46:09.684638 | finish at 2026-10-17 18:42:44 + [2026-10-14 14:00:14] iteration 1147/ 2384 | consumed samples: 36704 | elapsed time per iteration (ms): 220331.9 | throughput per GPU (TFLOP/s/GPU): 19.1 | MFU 6.13% | FLOPs estimator: workload-aware | logical tokens/s: 76145.1 | valid tokens/s: 76145.1 | physical tokens/s: 76721.8 | loss tokens/s: 55970.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.483879E-01 | lm-global-supervised-tokens: 1.233198E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.402253E-02 | lm-token-buffer-mib: 2.333714E+02 | global_load_balancing_loss: 1.002085E+00 | loss scale: 1.0 | grad norm: 0.108 | params norm: 11181.161 | params rms: 0.060542 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:42:30.547472 | finish at 2026-10-17 17:42:45 + [2026-10-14 14:04:02] iteration 1148/ 2384 | consumed samples: 36736 | elapsed time per iteration (ms): 227313.0 | throughput per GPU (TFLOP/s/GPU): 24.6 | MFU 7.90% | FLOPs estimator: workload-aware | logical tokens/s: 73806.5 | valid tokens/s: 73806.5 | physical tokens/s: 75375.5 | loss tokens/s: 56492.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.299155E-01 | lm-global-supervised-tokens: 1.284136E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.429439E-02 | lm-token-buffer-mib: 2.363279E+02 | global_load_balancing_loss: 1.000585E+00 | loss scale: 1.0 | grad norm: 0.258 | params norm: 11181.124 | params rms: 0.060542 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:02:38.851158 | finish at 2026-10-17 20:06:41 + [2026-10-14 14:07:43] iteration 1149/ 2384 | consumed samples: 36768 | elapsed time per iteration (ms): 221308.8 | throughput per GPU (TFLOP/s/GPU): 19.5 | MFU 6.24% | FLOPs estimator: workload-aware | logical tokens/s: 75809.0 | valid tokens/s: 75809.0 | physical tokens/s: 76714.5 | loss tokens/s: 56665.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.711078E-01 | lm-global-supervised-tokens: 1.254067E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.364505E-02 | lm-token-buffer-mib: 2.336713E+02 | global_load_balancing_loss: 9.992006E-01 | loss scale: 1.0 | grad norm: 0.369 | params norm: 11181.087 | params rms: 0.060542 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:55:16.388589 | finish at 2026-10-17 18:02:59 + [2026-10-14 14:11:48] iteration 1150/ 2384 | consumed samples: 36800 | elapsed time per iteration (ms): 244967.7 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.96% | FLOPs estimator: workload-aware | logical tokens/s: 68487.4 | valid tokens/s: 68487.4 | physical tokens/s: 69324.0 | loss tokens/s: 50763.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.103194E-01 | lm-global-supervised-tokens: 1.243533E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.618918E-02 | lm-token-buffer-mib: 2.344558E+02 | global_load_balancing_loss: 9.992638E-01 | loss scale: 1.0 | grad norm: 0.104 | params norm: 11181.049 | params rms: 0.060542 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:58:10.172403 | finish at 2026-10-18 02:09:58 + [2026-10-14 14:15:34] iteration 1151/ 2384 | consumed samples: 36832 | elapsed time per iteration (ms): 225762.7 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.57% | FLOPs estimator: workload-aware | logical tokens/s: 74313.5 | valid tokens/s: 74313.5 | physical tokens/s: 75228.0 | loss tokens/s: 56756.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.184976E-01 | lm-global-supervised-tokens: 1.281353E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.562224E-02 | lm-token-buffer-mib: 2.354133E+02 | global_load_balancing_loss: 9.991621E-01 | loss scale: 1.0 | grad norm: 0.165 | params norm: 11181.012 | params rms: 0.060542 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:19:25.427132 | finish at 2026-10-17 19:34:59 + [2026-10-14 14:19:17] iteration 1152/ 2384 | consumed samples: 36864 | elapsed time per iteration (ms): 223352.7 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.48% | FLOPs estimator: workload-aware | logical tokens/s: 75115.2 | valid tokens/s: 75115.2 | physical tokens/s: 75822.8 | loss tokens/s: 54785.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.233636E-01 | lm-global-supervised-tokens: 1.223649E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.779257E-02 | lm-token-buffer-mib: 2.350063E+02 | global_load_balancing_loss: 9.999164E-01 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11180.978 | params rms: 0.060541 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:26:10.569572 | finish at 2026-10-17 18:45:28 + [2026-10-14 14:22:58] iteration 1153/ 2384 | consumed samples: 36896 | elapsed time per iteration (ms): 220442.8 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.55% | FLOPs estimator: workload-aware | logical tokens/s: 76106.7 | valid tokens/s: 76106.7 | physical tokens/s: 76713.1 | loss tokens/s: 57688.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.264985E-01 | lm-global-supervised-tokens: 1.271710E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.337887E-02 | lm-token-buffer-mib: 2.347384E+02 | global_load_balancing_loss: 1.000317E+00 | loss scale: 1.0 | grad norm: 0.262 | params norm: 11180.947 | params rms: 0.060541 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:22:45.028450 | finish at 2026-10-17 17:45:43 + [2026-10-14 14:26:42] iteration 1154/ 2384 | consumed samples: 36928 | elapsed time per iteration (ms): 224625.0 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.13% | FLOPs estimator: workload-aware | logical tokens/s: 74689.8 | valid tokens/s: 74689.8 | physical tokens/s: 75491.9 | loss tokens/s: 54279.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.363433E-01 | lm-global-supervised-tokens: 1.219253E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.898862E-02 | lm-token-buffer-mib: 2.348145E+02 | global_load_balancing_loss: 1.000597E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11180.917 | params rms: 0.060541 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:44:48.772581 | finish at 2026-10-17 19:11:31 + [2026-10-14 14:30:26] iteration 1155/ 2384 | consumed samples: 36960 | elapsed time per iteration (ms): 223932.3 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.48% | FLOPs estimator: workload-aware | logical tokens/s: 74920.6 | valid tokens/s: 74920.6 | physical tokens/s: 75584.6 | loss tokens/s: 50735.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.473213E-01 | lm-global-supervised-tokens: 1.136131E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.374718E-02 | lm-token-buffer-mib: 2.333868E+02 | global_load_balancing_loss: 1.002134E+00 | loss scale: 1.0 | grad norm: 0.155 | params norm: 11180.888 | params rms: 0.060541 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:26:52.750515 | finish at 2026-10-17 18:57:19 + [2026-10-14 14:34:08] iteration 1156/ 2384 | consumed samples: 36992 | elapsed time per iteration (ms): 221673.4 | throughput per GPU (TFLOP/s/GPU): 24.1 | MFU 7.71% | FLOPs estimator: workload-aware | logical tokens/s: 75684.3 | valid tokens/s: 75684.3 | physical tokens/s: 77055.8 | loss tokens/s: 61182.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.588520E-01 | lm-global-supervised-tokens: 1.356245E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.896205E-02 | lm-token-buffer-mib: 2.329337E+02 | global_load_balancing_loss: 1.001740E+00 | loss scale: 1.0 | grad norm: 0.308 | params norm: 11180.856 | params rms: 0.060541 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:36:54.949747 | finish at 2026-10-17 18:11:03 + [2026-10-14 14:37:51] iteration 1157/ 2384 | consumed samples: 37024 | elapsed time per iteration (ms): 223050.7 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.87% | FLOPs estimator: workload-aware | logical tokens/s: 75216.8 | valid tokens/s: 75216.8 | physical tokens/s: 75594.4 | loss tokens/s: 58021.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.339541E-01 | lm-global-supervised-tokens: 1.294167E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.705617E-02 | lm-token-buffer-mib: 2.335808E+02 | global_load_balancing_loss: 1.000145E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11180.824 | params rms: 0.060541 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:01:23.158228 | finish at 2026-10-17 18:39:14 + [2026-10-14 14:41:36] iteration 1158/ 2384 | consumed samples: 37056 | elapsed time per iteration (ms): 225301.6 | throughput per GPU (TFLOP/s/GPU): 24.2 | MFU 7.75% | FLOPs estimator: workload-aware | logical tokens/s: 74465.4 | valid tokens/s: 74465.4 | physical tokens/s: 75316.7 | loss tokens/s: 56142.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.030265E-01 | lm-global-supervised-tokens: 1.264891E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.495431E-02 | lm-token-buffer-mib: 2.347270E+02 | global_load_balancing_loss: 1.000964E+00 | loss scale: 1.0 | grad norm: 0.752 | params norm: 11180.788 | params rms: 0.060540 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:43:39.718314 | finish at 2026-10-17 19:25:16 + [2026-10-14 14:45:20] iteration 1159/ 2384 | consumed samples: 37088 | elapsed time per iteration (ms): 223823.5 | throughput per GPU (TFLOP/s/GPU): 23.2 | MFU 7.45% | FLOPs estimator: workload-aware | logical tokens/s: 74957.3 | valid tokens/s: 74957.3 | physical tokens/s: 75855.8 | loss tokens/s: 57210.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.074791E-01 | lm-global-supervised-tokens: 1.280504E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.845989E-02 | lm-token-buffer-mib: 2.336324E+02 | global_load_balancing_loss: 9.999591E-01 | loss scale: 1.0 | grad norm: 0.219 | params norm: 11180.751 | params rms: 0.060540 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:09:43.747095 | finish at 2026-10-17 18:55:04 + [2026-10-14 14:49:28] iteration 1160/ 2384 | consumed samples: 37120 | elapsed time per iteration (ms): 247663.0 | throughput per GPU (TFLOP/s/GPU): 16.3 | MFU 5.22% | FLOPs estimator: workload-aware | logical tokens/s: 67742.1 | valid tokens/s: 67742.1 | physical tokens/s: 68982.5 | loss tokens/s: 55098.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.617164E-01 | lm-global-supervised-tokens: 1.364581E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.784115E-02 | lm-token-buffer-mib: 2.393279E+02 | global_load_balancing_loss: 9.997038E-01 | loss scale: 1.0 | grad norm: 0.615 | params norm: 11180.716 | params rms: 0.060540 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:12:19.456392 | finish at 2026-10-18 03:01:47 +times across ranks (ms): + save-checkpoint: + rank 0: 24825.88 + rank 1: 24825.90 + rank 2: 24825.90 + rank 3: 24825.89 + rank 4: 24825.91 + rank 5: 24825.92 + rank 6: 24825.92 + rank 7: 24825.91 + rank 8: 24826.02 + rank 9: 24826.03 + rank 10: 24826.07 + rank 11: 24826.06 + rank 12: 24826.03 + rank 13: 24825.99 + rank 14: 24826.04 + rank 15: 24826.00 + rank 16: 24825.18 + rank 17: 24825.18 + rank 18: 24825.18 + rank 19: 24825.17 + rank 20: 24825.16 + rank 21: 24825.15 + rank 22: 24826.32 + rank 23: 24825.17 + rank 24: 24825.47 + rank 25: 24825.46 + rank 26: 24826.21 + rank 27: 24825.49 + rank 28: 24825.41 + rank 29: 24825.47 + rank 30: 24825.51 + rank 31: 24825.43 + rank 32: 24826.01 + rank 33: 24825.99 + rank 34: 24825.82 + rank 35: 24826.03 + rank 36: 24825.97 + rank 37: 24825.95 + rank 38: 24825.80 + rank 39: 24825.65 + rank 40: 24825.96 + rank 41: 24825.93 + rank 42: 24825.93 + rank 43: 24825.95 + rank 44: 24825.92 + rank 45: 24825.93 + rank 46: 24825.93 + rank 47: 24825.92 + rank 48: 24825.79 + rank 49: 24825.81 + rank 50: 24825.80 + rank 51: 24825.78 + rank 52: 24825.89 + rank 53: 24825.90 + rank 54: 24825.83 + rank 55: 24825.87 + rank 56: 24826.17 + rank 57: 24826.19 + rank 58: 24826.19 + rank 59: 24826.23 + rank 60: 24826.18 + rank 61: 24826.16 + rank 62: 24826.17 + rank 63: 24826.16 + [2026-10-14 14:53:44] iteration 1161/ 2384 | consumed samples: 37152 | elapsed time per iteration (ms): 231760.0 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.14% | FLOPs estimator: workload-aware | logical tokens/s: 72390.3 | valid tokens/s: 72390.3 | physical tokens/s: 73091.6 | loss tokens/s: 56204.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.410467E-01 | lm-global-supervised-tokens: 1.302605E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.361528E-01 | lm-token-buffer-mib: 2.352966E+02 | global_load_balancing_loss: 9.994885E-01 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11180.681 | params rms: 0.060540 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:44:02.419338 | finish at 2026-10-17 21:37:47 + [2026-10-14 14:57:31] iteration 1162/ 2384 | consumed samples: 37184 | elapsed time per iteration (ms): 226304.3 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.01% | FLOPs estimator: workload-aware | logical tokens/s: 74135.6 | valid tokens/s: 74135.6 | physical tokens/s: 74673.0 | loss tokens/s: 55611.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.707312E-01 | lm-global-supervised-tokens: 1.258503E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.982325E-02 | lm-token-buffer-mib: 2.343383E+02 | global_load_balancing_loss: 9.979420E-01 | loss scale: 1.0 | grad norm: 0.124 | params norm: 11180.647 | params rms: 0.060540 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:49:03.836913 | finish at 2026-10-17 19:46:34 + [2026-10-14 15:01:14] iteration 1163/ 2384 | consumed samples: 37216 | elapsed time per iteration (ms): 223779.0 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.20% | FLOPs estimator: workload-aware | logical tokens/s: 74972.1 | valid tokens/s: 74972.1 | physical tokens/s: 75871.2 | loss tokens/s: 52704.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.741372E-01 | lm-global-supervised-tokens: 1.179411E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.040522E-02 | lm-token-buffer-mib: 2.344381E+02 | global_load_balancing_loss: 9.991575E-01 | loss scale: 1.0 | grad norm: 0.422 | params norm: 11180.612 | params rms: 0.060539 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:53:54.102578 | finish at 2026-10-17 18:55:08 + [2026-10-14 15:04:59] iteration 1164/ 2384 | consumed samples: 37248 | elapsed time per iteration (ms): 224408.8 | throughput per GPU (TFLOP/s/GPU): 24.2 | MFU 7.74% | FLOPs estimator: workload-aware | logical tokens/s: 74761.7 | valid tokens/s: 74761.7 | physical tokens/s: 75962.5 | loss tokens/s: 59612.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.190380E-01 | lm-global-supervised-tokens: 1.337761E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.573770E-02 | lm-token-buffer-mib: 2.359133E+02 | global_load_balancing_loss: 1.000873E+00 | loss scale: 1.0 | grad norm: 0.096 | params norm: 11180.580 | params rms: 0.060539 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:02:58.790836 | finish at 2026-10-17 19:07:58 + [2026-10-14 15:08:43] iteration 1165/ 2384 | consumed samples: 37280 | elapsed time per iteration (ms): 223906.0 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.11% | FLOPs estimator: workload-aware | logical tokens/s: 74929.7 | valid tokens/s: 74929.7 | physical tokens/s: 75648.9 | loss tokens/s: 53222.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.490978E-01 | lm-global-supervised-tokens: 1.191690E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.070821E-02 | lm-token-buffer-mib: 2.328503E+02 | global_load_balancing_loss: 1.002058E+00 | loss scale: 1.0 | grad norm: 0.195 | params norm: 11180.549 | params rms: 0.060539 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:49:01.444684 | finish at 2026-10-17 18:57:44 + [2026-10-14 15:12:26] iteration 1166/ 2384 | consumed samples: 37312 | elapsed time per iteration (ms): 223209.8 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.48% | FLOPs estimator: workload-aware | logical tokens/s: 75163.3 | valid tokens/s: 75163.3 | physical tokens/s: 75752.7 | loss tokens/s: 56790.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.037917E-01 | lm-global-supervised-tokens: 1.267621E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 6.610540E-02 | lm-token-buffer-mib: 2.346744E+02 | global_load_balancing_loss: 1.002737E+00 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11180.514 | params rms: 0.060539 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:31:09.502140 | finish at 2026-10-17 18:43:35 + [2026-10-14 15:16:08] iteration 1167/ 2384 | consumed samples: 37344 | elapsed time per iteration (ms): 222449.6 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.87% | FLOPs estimator: workload-aware | logical tokens/s: 75420.3 | valid tokens/s: 75420.2 | physical tokens/s: 75757.5 | loss tokens/s: 57997.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.123117E-01 | lm-global-supervised-tokens: 1.290162E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.718651E-02 | lm-token-buffer-mib: 2.329358E+02 | global_load_balancing_loss: 1.001419E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11180.474 | params rms: 0.060539 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:12:01.157664 | finish at 2026-10-17 18:28:09 + [2026-10-14 15:19:55] iteration 1168/ 2384 | consumed samples: 37376 | elapsed time per iteration (ms): 226788.6 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.44% | FLOPs estimator: workload-aware | logical tokens/s: 73977.3 | valid tokens/s: 73977.3 | physical tokens/s: 74812.4 | loss tokens/s: 54891.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.437182E-01 | lm-global-supervised-tokens: 1.244882E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.623556E-02 | lm-token-buffer-mib: 2.343251E+02 | global_load_balancing_loss: 1.000635E+00 | loss scale: 1.0 | grad norm: 0.297 | params norm: 11180.436 | params rms: 0.060538 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:36:14.914658 | finish at 2026-10-17 19:56:10 + [2026-10-14 15:23:40] iteration 1169/ 2384 | consumed samples: 37408 | elapsed time per iteration (ms): 224571.3 | throughput per GPU (TFLOP/s/GPU): 19.0 | MFU 6.09% | FLOPs estimator: workload-aware | logical tokens/s: 74707.7 | valid tokens/s: 74707.7 | physical tokens/s: 76088.5 | loss tokens/s: 55845.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.125821E-01 | lm-global-supervised-tokens: 1.254136E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.070070E-02 | lm-token-buffer-mib: 2.335924E+02 | global_load_balancing_loss: 9.999418E-01 | loss scale: 1.0 | grad norm: 0.226 | params norm: 11180.401 | params rms: 0.060538 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:47:34.069862 | finish at 2026-10-17 19:11:14 + [2026-10-14 15:27:44] iteration 1170/ 2384 | consumed samples: 37440 | elapsed time per iteration (ms): 244740.5 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.28% | FLOPs estimator: workload-aware | logical tokens/s: 68551.0 | valid tokens/s: 68551.0 | physical tokens/s: 68933.3 | loss tokens/s: 51302.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.044866E-01 | lm-global-supervised-tokens: 1.255584E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.249111E-02 | lm-token-buffer-mib: 2.338684E+02 | global_load_balancing_loss: 1.001531E+00 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11180.367 | params rms: 0.060538 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:31:55.017330 | finish at 2026-10-18 01:59:39 + [2026-10-14 15:31:29] iteration 1171/ 2384 | consumed samples: 37472 | elapsed time per iteration (ms): 224423.0 | throughput per GPU (TFLOP/s/GPU): 19.9 | MFU 6.39% | FLOPs estimator: workload-aware | logical tokens/s: 74757.1 | valid tokens/s: 74757.1 | physical tokens/s: 75765.8 | loss tokens/s: 55946.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.623828E-01 | lm-global-supervised-tokens: 1.255568E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.607698E-02 | lm-token-buffer-mib: 2.357033E+02 | global_load_balancing_loss: 9.987714E-01 | loss scale: 1.0 | grad norm: 0.192 | params norm: 11180.330 | params rms: 0.060538 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:37:05.144523 | finish at 2026-10-17 19:08:34 + [2026-10-14 15:35:11] iteration 1172/ 2384 | consumed samples: 37504 | elapsed time per iteration (ms): 222644.3 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.42% | FLOPs estimator: workload-aware | logical tokens/s: 75354.2 | valid tokens/s: 75354.2 | physical tokens/s: 75740.4 | loss tokens/s: 56526.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.382443E-01 | lm-global-supervised-tokens: 1.258538E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.350089E-02 | lm-token-buffer-mib: 2.326138E+02 | global_load_balancing_loss: 9.986102E-01 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11180.294 | params rms: 0.060538 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:57:24.900538 | finish at 2026-10-17 18:32:36 + [2026-10-14 15:38:54] iteration 1173/ 2384 | consumed samples: 37536 | elapsed time per iteration (ms): 222060.4 | throughput per GPU (TFLOP/s/GPU): 19.4 | MFU 6.22% | FLOPs estimator: workload-aware | logical tokens/s: 75552.4 | valid tokens/s: 75552.4 | physical tokens/s: 76321.7 | loss tokens/s: 56278.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.769115E-01 | lm-global-supervised-tokens: 1.249716E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.406098E-02 | lm-token-buffer-mib: 2.342860E+02 | global_load_balancing_loss: 1.000258E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11180.258 | params rms: 0.060537 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:41:55.120158 | finish at 2026-10-17 18:20:49 + [2026-10-14 15:42:35] iteration 1174/ 2384 | consumed samples: 37568 | elapsed time per iteration (ms): 221469.3 | throughput per GPU (TFLOP/s/GPU): 19.9 | MFU 6.37% | FLOPs estimator: workload-aware | logical tokens/s: 75754.1 | valid tokens/s: 75754.1 | physical tokens/s: 76165.6 | loss tokens/s: 55755.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.560747E-01 | lm-global-supervised-tokens: 1.234821E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.477192E-02 | lm-token-buffer-mib: 2.327155E+02 | global_load_balancing_loss: 1.001511E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11180.224 | params rms: 0.060537 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:26:17.907274 | finish at 2026-10-17 18:08:53 + [2026-10-14 15:46:16] iteration 1175/ 2384 | consumed samples: 37600 | elapsed time per iteration (ms): 220617.6 | throughput per GPU (TFLOP/s/GPU): 19.9 | MFU 6.39% | FLOPs estimator: workload-aware | logical tokens/s: 76046.5 | valid tokens/s: 76046.5 | physical tokens/s: 76623.0 | loss tokens/s: 58520.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.712650E-01 | lm-global-supervised-tokens: 1.291059E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.608872E-02 | lm-token-buffer-mib: 2.339614E+02 | global_load_balancing_loss: 1.001041E+00 | loss scale: 1.0 | grad norm: 0.122 | params norm: 11180.191 | params rms: 0.060537 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:05:26.678933 | finish at 2026-10-17 17:51:42 + [2026-10-14 15:49:59] iteration 1176/ 2384 | consumed samples: 37632 | elapsed time per iteration (ms): 223706.7 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.50% | FLOPs estimator: workload-aware | logical tokens/s: 74996.4 | valid tokens/s: 74996.4 | physical tokens/s: 75702.6 | loss tokens/s: 55083.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.348997E-01 | lm-global-supervised-tokens: 1.232247E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.815228E-02 | lm-token-buffer-mib: 2.348161E+02 | global_load_balancing_loss: 1.000576E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11180.158 | params rms: 0.060537 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:03:57.653095 | finish at 2026-10-17 18:53:57 + [2026-10-14 15:53:40] iteration 1177/ 2384 | consumed samples: 37664 | elapsed time per iteration (ms): 220502.6 | throughput per GPU (TFLOP/s/GPU): 25.3 | MFU 8.10% | FLOPs estimator: workload-aware | logical tokens/s: 76085.9 | valid tokens/s: 76085.9 | physical tokens/s: 76680.4 | loss tokens/s: 56848.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.328022E-01 | lm-global-supervised-tokens: 1.253531E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.816325E-02 | lm-token-buffer-mib: 2.337222E+02 | global_load_balancing_loss: 1.001055E+00 | loss scale: 1.0 | grad norm: 0.111 | params norm: 11180.123 | params rms: 0.060537 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:55:46.690807 | finish at 2026-10-17 17:49:27 + [2026-10-14 15:57:27] iteration 1178/ 2384 | consumed samples: 37696 | elapsed time per iteration (ms): 226701.0 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.34% | FLOPs estimator: workload-aware | logical tokens/s: 74005.8 | valid tokens/s: 74005.8 | physical tokens/s: 74938.9 | loss tokens/s: 51818.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.026974E-01 | lm-global-supervised-tokens: 1.174727E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.251570E-02 | lm-token-buffer-mib: 2.371241E+02 | global_load_balancing_loss: 1.001294E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11180.088 | params rms: 0.060537 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:56:41.452550 | finish at 2026-10-17 19:54:08 + [2026-10-14 16:01:09] iteration 1179/ 2384 | consumed samples: 37728 | elapsed time per iteration (ms): 222697.1 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.24% | FLOPs estimator: workload-aware | logical tokens/s: 75336.4 | valid tokens/s: 75336.4 | physical tokens/s: 76442.3 | loss tokens/s: 58980.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.407032E-01 | lm-global-supervised-tokens: 1.313470E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.791490E-02 | lm-token-buffer-mib: 2.341912E+02 | global_load_balancing_loss: 1.001554E+00 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11180.050 | params rms: 0.060536 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:32:30.065166 | finish at 2026-10-17 18:33:39 + [2026-10-14 16:05:13] iteration 1180/ 2384 | consumed samples: 37760 | elapsed time per iteration (ms): 243828.0 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.53% | FLOPs estimator: workload-aware | logical tokens/s: 68807.4 | valid tokens/s: 68807.4 | physical tokens/s: 69340.4 | loss tokens/s: 50389.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.165939E-01 | lm-global-supervised-tokens: 1.228635E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.755871E-02 | lm-token-buffer-mib: 2.346964E+02 | global_load_balancing_loss: 1.001030E+00 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11180.014 | params rms: 0.060536 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:32:48.929306 | finish at 2026-10-18 01:38:02 + [2026-10-14 16:08:52] iteration 1181/ 2384 | consumed samples: 37792 | elapsed time per iteration (ms): 218948.5 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.92% | FLOPs estimator: workload-aware | logical tokens/s: 76626.1 | valid tokens/s: 76626.1 | physical tokens/s: 76956.6 | loss tokens/s: 53191.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.636237E-01 | lm-global-supervised-tokens: 1.164628E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.669604E-02 | lm-token-buffer-mib: 2.324829E+02 | global_load_balancing_loss: 1.001598E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11179.981 | params rms: 0.060536 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:09:55.067773 | finish at 2026-10-17 17:18:47 + [2026-10-14 16:12:36] iteration 1182/ 2384 | consumed samples: 37824 | elapsed time per iteration (ms): 224012.1 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.66% | FLOPs estimator: workload-aware | logical tokens/s: 74894.2 | valid tokens/s: 74894.2 | physical tokens/s: 76052.8 | loss tokens/s: 60911.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.086634E-01 | lm-global-supervised-tokens: 1.364487E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.639668E-02 | lm-token-buffer-mib: 2.375621E+02 | global_load_balancing_loss: 1.001014E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11179.948 | params rms: 0.060536 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:47:42.486862 | finish at 2026-10-17 19:00:19 + [2026-10-14 16:16:19] iteration 1183/ 2384 | consumed samples: 37856 | elapsed time per iteration (ms): 222521.6 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.94% | FLOPs estimator: workload-aware | logical tokens/s: 75395.6 | valid tokens/s: 75395.6 | physical tokens/s: 75946.1 | loss tokens/s: 55332.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.625188E-01 | lm-global-supervised-tokens: 1.231274E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.620486E-02 | lm-token-buffer-mib: 2.326998E+02 | global_load_balancing_loss: 9.992399E-01 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11179.914 | params rms: 0.060536 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:14:08.459076 | finish at 2026-10-17 18:30:27 + [2026-10-14 16:20:01] iteration 1184/ 2384 | consumed samples: 37888 | elapsed time per iteration (ms): 222344.0 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.92% | FLOPs estimator: workload-aware | logical tokens/s: 75455.9 | valid tokens/s: 75455.9 | physical tokens/s: 76439.1 | loss tokens/s: 60081.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.819687E-01 | lm-global-supervised-tokens: 1.335866E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.379441E-02 | lm-token-buffer-mib: 2.341783E+02 | global_load_balancing_loss: 1.000514E+00 | loss scale: 1.0 | grad norm: 0.177 | params norm: 11179.880 | params rms: 0.060535 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:06:52.823009 | finish at 2026-10-17 18:26:54 + [2026-10-14 16:23:46] iteration 1185/ 2384 | consumed samples: 37920 | elapsed time per iteration (ms): 225120.4 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.09% | FLOPs estimator: workload-aware | logical tokens/s: 74525.3 | valid tokens/s: 74525.3 | physical tokens/s: 74748.4 | loss tokens/s: 52983.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.900678E-01 | lm-global-supervised-tokens: 1.192772E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.744428E-02 | lm-token-buffer-mib: 2.325406E+02 | global_load_balancing_loss: 1.000651E+00 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11179.845 | params rms: 0.060535 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:58:39.337531 | finish at 2026-10-17 19:22:25 + [2026-10-14 16:27:31] iteration 1186/ 2384 | consumed samples: 37952 | elapsed time per iteration (ms): 225156.7 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.30% | FLOPs estimator: workload-aware | logical tokens/s: 74513.4 | valid tokens/s: 74513.4 | physical tokens/s: 74753.7 | loss tokens/s: 58089.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.518036E-01 | lm-global-supervised-tokens: 1.307912E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.722190E-02 | lm-token-buffer-mib: 2.328226E+02 | global_load_balancing_loss: 9.997135E-01 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11179.813 | params rms: 0.060535 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:55:37.730474 | finish at 2026-10-17 19:23:09 + [2026-10-14 16:31:16] iteration 1187/ 2384 | consumed samples: 37984 | elapsed time per iteration (ms): 224858.9 | throughput per GPU (TFLOP/s/GPU): 23.7 | MFU 7.58% | FLOPs estimator: workload-aware | logical tokens/s: 74612.0 | valid tokens/s: 74612.0 | physical tokens/s: 75520.4 | loss tokens/s: 54051.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.828751E-01 | lm-global-supervised-tokens: 1.215385E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.457452E-02 | lm-token-buffer-mib: 2.360953E+02 | global_load_balancing_loss: 1.000076E+00 | loss scale: 1.0 | grad norm: 0.146 | params norm: 11179.777 | params rms: 0.060535 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:45:56.152471 | finish at 2026-10-17 19:17:12 + [2026-10-14 16:34:59] iteration 1188/ 2384 | consumed samples: 38016 | elapsed time per iteration (ms): 222915.9 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.88% | FLOPs estimator: workload-aware | logical tokens/s: 75262.5 | valid tokens/s: 75262.5 | physical tokens/s: 76165.8 | loss tokens/s: 59494.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.134053E-01 | lm-global-supervised-tokens: 1.326218E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.404674E-02 | lm-token-buffer-mib: 2.333572E+02 | global_load_balancing_loss: 1.000155E+00 | loss scale: 1.0 | grad norm: 0.152 | params norm: 11179.738 | params rms: 0.060535 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:03:27.402133 | finish at 2026-10-17 18:38:26 + [2026-10-14 16:38:46] iteration 1189/ 2384 | consumed samples: 38048 | elapsed time per iteration (ms): 226980.0 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.17% | FLOPs estimator: workload-aware | logical tokens/s: 73914.9 | valid tokens/s: 73914.9 | physical tokens/s: 74260.6 | loss tokens/s: 56633.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.073934E-01 | lm-global-supervised-tokens: 1.285467E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.724665E-02 | lm-token-buffer-mib: 2.335326E+02 | global_load_balancing_loss: 1.002155E+00 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11179.701 | params rms: 0.060534 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:20:41.089481 | finish at 2026-10-17 19:59:27 + [2026-10-14 16:42:48] iteration 1190/ 2384 | consumed samples: 38080 | elapsed time per iteration (ms): 241987.0 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.93% | FLOPs estimator: workload-aware | logical tokens/s: 69331.0 | valid tokens/s: 69331.0 | physical tokens/s: 69631.8 | loss tokens/s: 50832.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.086575E-01 | lm-global-supervised-tokens: 1.230079E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.455717E-02 | lm-token-buffer-mib: 2.328443E+02 | global_load_balancing_loss: 1.001113E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11179.663 | params rms: 0.060534 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:15:32.470870 | finish at 2026-10-18 00:58:20 + [2026-10-14 16:46:30] iteration 1191/ 2384 | consumed samples: 38112 | elapsed time per iteration (ms): 221965.7 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.62% | FLOPs estimator: workload-aware | logical tokens/s: 75584.5 | valid tokens/s: 75584.5 | physical tokens/s: 76444.8 | loss tokens/s: 54973.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.240275E-01 | lm-global-supervised-tokens: 1.220226E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.365102E-02 | lm-token-buffer-mib: 2.339290E+02 | global_load_balancing_loss: 1.000714E+00 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11179.627 | params rms: 0.060534 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:33:25.065203 | finish at 2026-10-17 18:19:55 + [2026-10-14 16:50:13] iteration 1192/ 2384 | consumed samples: 38144 | elapsed time per iteration (ms): 223387.4 | throughput per GPU (TFLOP/s/GPU): 24.9 | MFU 7.97% | FLOPs estimator: workload-aware | logical tokens/s: 75103.6 | valid tokens/s: 75103.6 | physical tokens/s: 75648.6 | loss tokens/s: 59091.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.197039E-01 | lm-global-supervised-tokens: 1.320032E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.906828E-02 | lm-token-buffer-mib: 2.336006E+02 | global_load_balancing_loss: 1.000874E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11179.590 | params rms: 0.060534 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:57:57.726698 | finish at 2026-10-17 18:48:11 + [2026-10-14 16:53:56] iteration 1193/ 2384 | consumed samples: 38176 | elapsed time per iteration (ms): 223205.0 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.46% | FLOPs estimator: workload-aware | logical tokens/s: 75164.9 | valid tokens/s: 75164.9 | physical tokens/s: 76029.5 | loss tokens/s: 60795.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.604229E-01 | lm-global-supervised-tokens: 1.356978E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.276848E-02 | lm-token-buffer-mib: 2.338818E+02 | global_load_balancing_loss: 1.000587E+00 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11179.555 | params rms: 0.060534 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:50:37.188984 | finish at 2026-10-17 18:44:34 + [2026-10-14 16:57:40] iteration 1194/ 2384 | consumed samples: 38208 | elapsed time per iteration (ms): 223809.1 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.22% | FLOPs estimator: workload-aware | logical tokens/s: 74961.9 | valid tokens/s: 74961.9 | physical tokens/s: 75640.2 | loss tokens/s: 55616.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.003845E-01 | lm-global-supervised-tokens: 1.244750E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.664330E-02 | lm-token-buffer-mib: 2.356501E+02 | global_load_balancing_loss: 1.001276E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11179.525 | params rms: 0.060533 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:58:52.853580 | finish at 2026-10-17 18:56:33 + [2026-10-14 17:01:26] iteration 1195/ 2384 | consumed samples: 38240 | elapsed time per iteration (ms): 225899.5 | throughput per GPU (TFLOP/s/GPU): 25.2 | MFU 8.09% | FLOPs estimator: workload-aware | logical tokens/s: 74268.3 | valid tokens/s: 74268.3 | physical tokens/s: 74752.4 | loss tokens/s: 56139.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 9.147135E-01 | lm-global-supervised-tokens: 1.268189E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.515220E-02 | lm-token-buffer-mib: 2.329504E+02 | global_load_balancing_loss: 9.997327E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11179.498 | params rms: 0.060533 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:36:34.530036 | finish at 2026-10-17 19:38:01 + [2026-10-14 17:05:07] iteration 1196/ 2384 | consumed samples: 38272 | elapsed time per iteration (ms): 221276.8 | throughput per GPU (TFLOP/s/GPU): 23.9 | MFU 7.68% | FLOPs estimator: workload-aware | logical tokens/s: 75819.8 | valid tokens/s: 75819.8 | physical tokens/s: 76218.6 | loss tokens/s: 51460.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.123810E-01 | lm-global-supervised-tokens: 1.138706E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.826312E-02 | lm-token-buffer-mib: 2.327356E+02 | global_load_balancing_loss: 9.986256E-01 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11179.468 | params rms: 0.060533 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:01:16.813943 | finish at 2026-10-17 18:06:24 + [2026-10-14 17:08:51] iteration 1197/ 2384 | consumed samples: 38304 | elapsed time per iteration (ms): 223357.1 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 7.00% | FLOPs estimator: workload-aware | logical tokens/s: 75113.7 | valid tokens/s: 75113.7 | physical tokens/s: 75863.5 | loss tokens/s: 55703.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.266836E-01 | lm-global-supervised-tokens: 1.244168E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 6.500621E-02 | lm-token-buffer-mib: 2.363567E+02 | global_load_balancing_loss: 9.995536E-01 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11179.433 | params rms: 0.060533 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:38:44.830167 | finish at 2026-10-17 18:47:36 + [2026-10-14 17:12:34] iteration 1198/ 2384 | consumed samples: 38336 | elapsed time per iteration (ms): 222981.1 | throughput per GPU (TFLOP/s/GPU): 19.6 | MFU 6.27% | FLOPs estimator: workload-aware | logical tokens/s: 75240.5 | valid tokens/s: 75240.5 | physical tokens/s: 75657.6 | loss tokens/s: 58165.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.267778E-01 | lm-global-supervised-tokens: 1.296972E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.420456E-02 | lm-token-buffer-mib: 2.327872E+02 | global_load_balancing_loss: 9.998555E-01 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11179.401 | params rms: 0.060533 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:27:35.529559 | finish at 2026-10-17 18:40:09 + [2026-10-14 17:16:20] iteration 1199/ 2384 | consumed samples: 38368 | elapsed time per iteration (ms): 225970.3 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.20% | FLOPs estimator: workload-aware | logical tokens/s: 74245.1 | valid tokens/s: 74245.1 | physical tokens/s: 75271.6 | loss tokens/s: 54879.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.309614E-01 | lm-global-supervised-tokens: 1.240120E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.946525E-02 | lm-token-buffer-mib: 2.353001E+02 | global_load_balancing_loss: 1.000136E+00 | loss scale: 1.0 | grad norm: 0.151 | params norm: 11179.371 | params rms: 0.060533 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:22:54.785957 | finish at 2026-10-17 19:39:15 + [2026-10-14 17:20:27] iteration 1200/ 2384 | consumed samples: 38400 | elapsed time per iteration (ms): 246812.3 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.81% | FLOPs estimator: workload-aware | logical tokens/s: 67975.4 | valid tokens/s: 67975.4 | physical tokens/s: 68604.7 | loss tokens/s: 51951.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.417962E-01 | lm-global-supervised-tokens: 1.282229E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.957560E-02 | lm-token-buffer-mib: 2.354019E+02 | global_load_balancing_loss: 1.001145E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11179.342 | params rms: 0.060533 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:10:25.749611 | finish at 2026-10-18 02:30:52 +times across ranks (ms): + save-checkpoint: + rank 0: 24728.38 + rank 1: 24728.37 + rank 2: 24728.43 + rank 3: 24728.42 + rank 4: 24728.40 + rank 5: 24728.40 + rank 6: 24728.41 + rank 7: 24728.40 + rank 8: 24727.82 + rank 9: 24727.85 + rank 10: 24727.86 + rank 11: 24727.84 + rank 12: 24727.82 + rank 13: 24727.82 + rank 14: 24727.83 + rank 15: 24727.83 + rank 16: 24728.56 + rank 17: 24728.56 + rank 18: 24728.54 + rank 19: 24728.55 + rank 20: 24728.53 + rank 21: 24728.52 + rank 22: 24728.60 + rank 23: 24728.52 + rank 24: 24728.53 + rank 25: 24728.50 + rank 26: 24728.54 + rank 27: 24728.56 + rank 28: 24728.50 + rank 29: 24728.54 + rank 30: 24728.59 + rank 31: 24728.51 + rank 32: 24728.52 + rank 33: 24728.54 + rank 34: 24728.53 + rank 35: 24728.55 + rank 36: 24728.48 + rank 37: 24728.49 + rank 38: 24728.49 + rank 39: 24728.49 + rank 40: 24728.20 + rank 41: 24728.20 + rank 42: 24728.19 + rank 43: 24728.21 + rank 44: 24728.19 + rank 45: 24728.20 + rank 46: 24728.28 + rank 47: 24728.18 + rank 48: 24728.35 + rank 49: 24728.35 + rank 50: 24728.40 + rank 51: 24728.35 + rank 52: 24728.34 + rank 53: 24728.36 + rank 54: 24728.34 + rank 55: 24728.50 + rank 56: 24728.38 + rank 57: 24728.37 + rank 58: 24728.38 + rank 59: 24728.37 + rank 60: 24728.39 + rank 61: 24728.37 + rank 62: 24728.36 + rank 63: 24728.38 + [2026-10-14 17:24:30] iteration 1201/ 2384 | consumed samples: 38432 | elapsed time per iteration (ms): 218134.8 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.94% | FLOPs estimator: workload-aware | logical tokens/s: 76912.0 | valid tokens/s: 76912.0 | physical tokens/s: 77543.2 | loss tokens/s: 58053.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.640007E-01 | lm-global-supervised-tokens: 1.266342E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.409846E-01 | lm-token-buffer-mib: 2.334067E+02 | global_load_balancing_loss: 1.001387E+00 | loss scale: 1.0 | grad norm: 0.198 | params norm: 11179.312 | params rms: 0.060532 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:40:53.515734 | finish at 2026-10-17 17:05:23 + [2026-10-14 17:28:16] iteration 1202/ 2384 | consumed samples: 38464 | elapsed time per iteration (ms): 226564.0 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.40% | FLOPs estimator: workload-aware | logical tokens/s: 74050.5 | valid tokens/s: 74050.5 | physical tokens/s: 74628.1 | loss tokens/s: 53484.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.568495E-01 | lm-global-supervised-tokens: 1.211765E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.406286E-02 | lm-token-buffer-mib: 2.336943E+02 | global_load_balancing_loss: 1.001873E+00 | loss scale: 1.0 | grad norm: 0.116 | params norm: 11179.278 | params rms: 0.060532 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:23:18.638572 | finish at 2026-10-17 19:51:35 + [2026-10-14 17:31:57] iteration 1203/ 2384 | consumed samples: 38496 | elapsed time per iteration (ms): 221166.6 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.56% | FLOPs estimator: workload-aware | logical tokens/s: 75857.8 | valid tokens/s: 75857.8 | physical tokens/s: 76697.3 | loss tokens/s: 59538.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.405604E-01 | lm-global-supervised-tokens: 1.316791E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.395875E-02 | lm-token-buffer-mib: 2.347876E+02 | global_load_balancing_loss: 1.001032E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11179.241 | params rms: 0.060532 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 0:33:17.696865 | finish at 2026-10-17 18:05:15 + [2026-10-14 17:35:45] iteration 1204/ 2384 | consumed samples: 38528 | elapsed time per iteration (ms): 227453.9 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.76% | FLOPs estimator: workload-aware | logical tokens/s: 73760.9 | valid tokens/s: 73760.9 | physical tokens/s: 74753.6 | loss tokens/s: 56501.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.612304E-01 | lm-global-supervised-tokens: 1.285146E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.387241E-02 | lm-token-buffer-mib: 2.373554E+02 | global_load_balancing_loss: 1.000927E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11179.204 | params rms: 0.060532 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:33:15.638409 | finish at 2026-10-17 20:09:00 + [2026-10-14 17:39:28] iteration 1205/ 2384 | consumed samples: 38560 | elapsed time per iteration (ms): 223360.8 | throughput per GPU (TFLOP/s/GPU): 18.8 | MFU 6.04% | FLOPs estimator: workload-aware | logical tokens/s: 75112.5 | valid tokens/s: 75112.5 | physical tokens/s: 75592.5 | loss tokens/s: 55349.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.415148E-01 | lm-global-supervised-tokens: 1.236293E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.327399E-02 | lm-token-buffer-mib: 2.331604E+02 | global_load_balancing_loss: 1.000473E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11179.173 | params rms: 0.060532 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:09:02.440014 | finish at 2026-10-17 18:48:31 + [2026-10-14 17:43:11] iteration 1206/ 2384 | consumed samples: 38592 | elapsed time per iteration (ms): 222970.0 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.96% | FLOPs estimator: workload-aware | logical tokens/s: 75244.1 | valid tokens/s: 75244.1 | physical tokens/s: 76079.0 | loss tokens/s: 54812.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.656787E-01 | lm-global-supervised-tokens: 1.222161E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.849872E-02 | lm-token-buffer-mib: 2.349250E+02 | global_load_balancing_loss: 9.993583E-01 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11179.143 | params rms: 0.060531 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 0:57:38.608637 | finish at 2026-10-17 18:40:50 + [2026-10-14 17:46:54] iteration 1207/ 2384 | consumed samples: 38624 | elapsed time per iteration (ms): 222690.0 | throughput per GPU (TFLOP/s/GPU): 25.1 | MFU 8.03% | FLOPs estimator: workload-aware | logical tokens/s: 75338.7 | valid tokens/s: 75338.7 | physical tokens/s: 76062.5 | loss tokens/s: 56538.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.670893E-01 | lm-global-supervised-tokens: 1.259054E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.156962E-02 | lm-token-buffer-mib: 2.336432E+02 | global_load_balancing_loss: 9.990898E-01 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11179.109 | params rms: 0.060531 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 0:48:26.183946 | finish at 2026-10-17 18:35:20 + [2026-10-14 17:50:38] iteration 1208/ 2384 | consumed samples: 38656 | elapsed time per iteration (ms): 224233.4 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.38% | FLOPs estimator: workload-aware | logical tokens/s: 74820.3 | valid tokens/s: 74820.3 | physical tokens/s: 75568.1 | loss tokens/s: 60772.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.548364E-01 | lm-global-supervised-tokens: 1.362727E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.464460E-02 | lm-token-buffer-mib: 2.345916E+02 | global_load_balancing_loss: 1.000539E+00 | loss scale: 1.0 | grad norm: 0.100 | params norm: 11179.073 | params rms: 0.060531 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:14:58.446562 | finish at 2026-10-17 19:05:37 + [2026-10-14 17:54:22] iteration 1209/ 2384 | consumed samples: 38688 | elapsed time per iteration (ms): 224140.1 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.38% | FLOPs estimator: workload-aware | logical tokens/s: 74851.3 | valid tokens/s: 74851.3 | physical tokens/s: 76088.4 | loss tokens/s: 57719.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.691324E-01 | lm-global-supervised-tokens: 1.293724E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.978322E-02 | lm-token-buffer-mib: 2.343004E+02 | global_load_balancing_loss: 1.000634E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11179.037 | params rms: 0.060531 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:09:24.580244 | finish at 2026-10-17 19:03:47 + [2026-10-14 17:58:24] iteration 1210/ 2384 | consumed samples: 38720 | elapsed time per iteration (ms): 242179.6 | throughput per GPU (TFLOP/s/GPU): 18.8 | MFU 6.02% | FLOPs estimator: workload-aware | logical tokens/s: 69275.8 | valid tokens/s: 69275.8 | physical tokens/s: 69670.2 | loss tokens/s: 47354.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.580021E-01 | lm-global-supervised-tokens: 1.146824E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.753217E-02 | lm-token-buffer-mib: 2.329861E+02 | global_load_balancing_loss: 1.001199E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11179.002 | params rms: 0.060531 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:58:38.908900 | finish at 2026-10-18 00:57:03 + [2026-10-14 18:02:08] iteration 1211/ 2384 | consumed samples: 38752 | elapsed time per iteration (ms): 224073.4 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.27% | FLOPs estimator: workload-aware | logical tokens/s: 74873.7 | valid tokens/s: 74873.7 | physical tokens/s: 75630.9 | loss tokens/s: 59013.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.681439E-01 | lm-global-supervised-tokens: 1.322339E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.668071E-02 | lm-token-buffer-mib: 2.339035E+02 | global_load_balancing_loss: 1.002257E+00 | loss scale: 1.0 | grad norm: 0.169 | params norm: 11178.968 | params rms: 0.060530 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:00:38.073614 | finish at 2026-10-17 19:02:47 + [2026-10-14 18:05:51] iteration 1212/ 2384 | consumed samples: 38784 | elapsed time per iteration (ms): 222635.8 | throughput per GPU (TFLOP/s/GPU): 19.3 | MFU 6.19% | FLOPs estimator: workload-aware | logical tokens/s: 75357.2 | valid tokens/s: 75357.2 | physical tokens/s: 76486.7 | loss tokens/s: 56041.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.326868E-01 | lm-global-supervised-tokens: 1.247676E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.930829E-02 | lm-token-buffer-mib: 2.366526E+02 | global_load_balancing_loss: 9.997529E-01 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11178.933 | params rms: 0.060530 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 0:28:49.119742 | finish at 2026-10-17 18:34:40 + [2026-10-14 18:09:40] iteration 1213/ 2384 | consumed samples: 38816 | elapsed time per iteration (ms): 228862.8 | throughput per GPU (TFLOP/s/GPU): 28.3 | MFU 9.09% | FLOPs estimator: workload-aware | logical tokens/s: 73306.5 | valid tokens/s: 73306.5 | physical tokens/s: 74074.2 | loss tokens/s: 54296.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.531867E-01 | lm-global-supervised-tokens: 1.242637E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.716827E-02 | lm-token-buffer-mib: 2.347820E+02 | global_load_balancing_loss: 9.992374E-01 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11178.896 | params rms: 0.060530 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:26:38.327216 | finish at 2026-10-17 20:36:18 + [2026-10-14 18:13:23] iteration 1214/ 2384 | consumed samples: 38848 | elapsed time per iteration (ms): 222999.7 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.18% | FLOPs estimator: workload-aware | logical tokens/s: 75234.2 | valid tokens/s: 75234.2 | physical tokens/s: 75852.4 | loss tokens/s: 59551.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.087375E-01 | lm-global-supervised-tokens: 1.327991E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.309246E-02 | lm-token-buffer-mib: 2.328928E+02 | global_load_balancing_loss: 9.988249E-01 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11178.855 | params rms: 0.060530 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 0:28:29.639039 | finish at 2026-10-17 18:41:53 + [2026-10-14 18:17:09] iteration 1215/ 2384 | consumed samples: 38880 | elapsed time per iteration (ms): 225694.4 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.91% | FLOPs estimator: workload-aware | logical tokens/s: 74335.8 | valid tokens/s: 74335.8 | physical tokens/s: 75126.3 | loss tokens/s: 54672.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.798705E-01 | lm-global-supervised-tokens: 1.233936E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.477212E-02 | lm-token-buffer-mib: 2.345316E+02 | global_load_balancing_loss: 1.000020E+00 | loss scale: 1.0 | grad norm: 0.164 | params norm: 11178.815 | params rms: 0.060530 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:17:16.791032 | finish at 2026-10-17 19:34:25 + [2026-10-14 18:20:49] iteration 1216/ 2384 | consumed samples: 38912 | elapsed time per iteration (ms): 220777.8 | throughput per GPU (TFLOP/s/GPU): 24.5 | MFU 7.84% | FLOPs estimator: workload-aware | logical tokens/s: 75991.3 | valid tokens/s: 75991.3 | physical tokens/s: 76424.8 | loss tokens/s: 54068.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.225710E-01 | lm-global-supervised-tokens: 1.193701E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.310190E-02 | lm-token-buffer-mib: 2.338266E+02 | global_load_balancing_loss: 1.000463E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11178.772 | params rms: 0.060529 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:37:48.487206 | finish at 2026-10-17 17:58:38 + [2026-10-14 18:24:30] iteration 1217/ 2384 | consumed samples: 38944 | elapsed time per iteration (ms): 220197.6 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.01% | FLOPs estimator: workload-aware | logical tokens/s: 76191.5 | valid tokens/s: 76191.5 | physical tokens/s: 76463.1 | loss tokens/s: 58867.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.584066E-01 | lm-global-supervised-tokens: 1.296255E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.880610E-02 | lm-token-buffer-mib: 2.330199E+02 | global_load_balancing_loss: 1.000412E+00 | loss scale: 1.0 | grad norm: 0.195 | params norm: 11178.730 | params rms: 0.060529 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:22:50.647760 | finish at 2026-10-17 17:47:20 + [2026-10-14 18:28:14] iteration 1218/ 2384 | consumed samples: 38976 | elapsed time per iteration (ms): 224740.9 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.54% | FLOPs estimator: workload-aware | logical tokens/s: 74651.3 | valid tokens/s: 74651.3 | physical tokens/s: 75556.7 | loss tokens/s: 59865.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.003392E-01 | lm-global-supervised-tokens: 1.345422E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.456479E-02 | lm-token-buffer-mib: 2.340899E+02 | global_load_balancing_loss: 1.000348E+00 | loss scale: 1.0 | grad norm: 0.261 | params norm: 11178.687 | params rms: 0.060529 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 0:47:27.886666 | finish at 2026-10-17 19:15:42 + [2026-10-14 18:31:59] iteration 1219/ 2384 | consumed samples: 39008 | elapsed time per iteration (ms): 224628.0 | throughput per GPU (TFLOP/s/GPU): 18.1 | MFU 5.81% | FLOPs estimator: workload-aware | logical tokens/s: 74688.9 | valid tokens/s: 74688.9 | physical tokens/s: 76577.3 | loss tokens/s: 60726.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.449032E-01 | lm-global-supervised-tokens: 1.364075E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.964252E-02 | lm-token-buffer-mib: 2.350914E+02 | global_load_balancing_loss: 1.002329E+00 | loss scale: 1.0 | grad norm: 1.224 | params norm: 11178.644 | params rms: 0.060529 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 0:41:31.570861 | finish at 2026-10-17 19:13:31 + [2026-10-14 18:36:03] iteration 1220/ 2384 | consumed samples: 39040 | elapsed time per iteration (ms): 244074.9 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.73% | FLOPs estimator: workload-aware | logical tokens/s: 68737.8 | valid tokens/s: 68737.8 | physical tokens/s: 69320.4 | loss tokens/s: 50931.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.153302E-01 | lm-global-supervised-tokens: 1.243110E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.759650E-02 | lm-token-buffer-mib: 2.341898E+02 | global_load_balancing_loss: 1.001359E+00 | loss scale: 1.0 | grad norm: 0.493 | params norm: 11178.599 | params rms: 0.060528 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:55:03.137152 | finish at 2026-10-18 01:31:06 + [2026-10-14 18:39:46] iteration 1221/ 2384 | consumed samples: 39072 | elapsed time per iteration (ms): 223286.2 | throughput per GPU (TFLOP/s/GPU): 26.0 | MFU 8.32% | FLOPs estimator: workload-aware | logical tokens/s: 75137.6 | valid tokens/s: 75137.6 | physical tokens/s: 75559.6 | loss tokens/s: 53951.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.763067E-01 | lm-global-supervised-tokens: 1.204666E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.366898E-02 | lm-token-buffer-mib: 2.330252E+02 | global_load_balancing_loss: 1.002579E+00 | loss scale: 1.0 | grad norm: 0.202 | params norm: 11178.557 | params rms: 0.060528 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 0:08:01.838454 | finish at 2026-10-17 18:47:48 + [2026-10-14 18:43:25] iteration 1222/ 2384 | consumed samples: 39104 | elapsed time per iteration (ms): 218826.0 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.62% | FLOPs estimator: workload-aware | logical tokens/s: 76669.1 | valid tokens/s: 76669.0 | physical tokens/s: 77526.4 | loss tokens/s: 60024.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.708276E-01 | lm-global-supervised-tokens: 1.313499E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.510650E-02 | lm-token-buffer-mib: 2.334317E+02 | global_load_balancing_loss: 1.000172E+00 | loss scale: 1.0 | grad norm: 0.113 | params norm: 11178.517 | params rms: 0.060528 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:37:55.865440 | finish at 2026-10-17 17:21:21 + [2026-10-14 18:47:07] iteration 1223/ 2384 | consumed samples: 39136 | elapsed time per iteration (ms): 221835.6 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.64% | FLOPs estimator: workload-aware | logical tokens/s: 75629.0 | valid tokens/s: 75629.0 | physical tokens/s: 76370.5 | loss tokens/s: 58004.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.082396E-01 | lm-global-supervised-tokens: 1.286735E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.928480E-02 | lm-token-buffer-mib: 2.347831E+02 | global_load_balancing_loss: 1.001263E+00 | loss scale: 1.0 | grad norm: 0.184 | params norm: 11178.479 | params rms: 0.060528 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:32:31.159440 | finish at 2026-10-17 18:19:38 + [2026-10-14 18:50:56] iteration 1224/ 2384 | consumed samples: 39168 | elapsed time per iteration (ms): 229363.4 | throughput per GPU (TFLOP/s/GPU): 25.0 | MFU 8.00% | FLOPs estimator: workload-aware | logical tokens/s: 73146.7 | valid tokens/s: 73146.7 | physical tokens/s: 74400.6 | loss tokens/s: 56398.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.144387E-01 | lm-global-supervised-tokens: 1.293569E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.503353E-02 | lm-token-buffer-mib: 2.378778E+02 | global_load_balancing_loss: 1.000342E+00 | loss scale: 1.0 | grad norm: 0.159 | params norm: 11178.443 | params rms: 0.060528 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:54:21.520748 | finish at 2026-10-17 20:45:18 + [2026-10-14 18:54:40] iteration 1225/ 2384 | consumed samples: 39200 | elapsed time per iteration (ms): 223266.1 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.11% | FLOPs estimator: workload-aware | logical tokens/s: 75144.4 | valid tokens/s: 75144.4 | physical tokens/s: 76207.7 | loss tokens/s: 55793.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.298122E-01 | lm-global-supervised-tokens: 1.245688E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.869614E-02 | lm-token-buffer-mib: 2.348870E+02 | global_load_balancing_loss: 1.000154E+00 | loss scale: 1.0 | grad norm: 0.187 | params norm: 11178.406 | params rms: 0.060527 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:52:45.378317 | finish at 2026-10-17 18:47:25 + [2026-10-14 18:58:25] iteration 1226/ 2384 | consumed samples: 39232 | elapsed time per iteration (ms): 225274.8 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.83% | FLOPs estimator: workload-aware | logical tokens/s: 74474.2 | valid tokens/s: 74474.2 | physical tokens/s: 76234.2 | loss tokens/s: 57021.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.757579E-01 | lm-global-supervised-tokens: 1.284551E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.821430E-02 | lm-token-buffer-mib: 2.380683E+02 | global_load_balancing_loss: 9.979057E-01 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11178.370 | params rms: 0.060527 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 0:27:48.169328 | finish at 2026-10-17 19:26:13 + [2026-10-14 19:02:09] iteration 1227/ 2384 | consumed samples: 39264 | elapsed time per iteration (ms): 224031.2 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.99% | FLOPs estimator: workload-aware | logical tokens/s: 74887.8 | valid tokens/s: 74887.8 | physical tokens/s: 75727.5 | loss tokens/s: 56939.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.241657E-01 | lm-global-supervised-tokens: 1.275615E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.488497E-02 | lm-token-buffer-mib: 2.350579E+02 | global_load_balancing_loss: 1.000954E+00 | loss scale: 1.0 | grad norm: 0.180 | params norm: 11178.338 | params rms: 0.060527 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 0:00:04.047841 | finish at 2026-10-17 19:02:13 + [2026-10-14 19:05:52] iteration 1228/ 2384 | consumed samples: 39296 | elapsed time per iteration (ms): 223314.7 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.60% | FLOPs estimator: workload-aware | logical tokens/s: 75128.0 | valid tokens/s: 75128.0 | physical tokens/s: 75575.2 | loss tokens/s: 55051.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.340948E-01 | lm-global-supervised-tokens: 1.229376E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.325000E-02 | lm-token-buffer-mib: 2.332560E+02 | global_load_balancing_loss: 9.981987E-01 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11178.309 | params rms: 0.060527 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:42:31.837169 | finish at 2026-10-17 18:48:24 + [2026-10-14 19:09:33] iteration 1229/ 2384 | consumed samples: 39328 | elapsed time per iteration (ms): 220302.4 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.41% | FLOPs estimator: workload-aware | logical tokens/s: 76155.4 | valid tokens/s: 76155.4 | physical tokens/s: 76668.5 | loss tokens/s: 57754.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.633137E-01 | lm-global-supervised-tokens: 1.272345E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.397380E-02 | lm-token-buffer-mib: 2.336131E+02 | global_load_balancing_loss: 9.982369E-01 | loss scale: 1.0 | grad norm: 0.136 | params norm: 11178.280 | params rms: 0.060527 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:40:49.217606 | finish at 2026-10-17 17:50:22 + [2026-10-14 19:13:36] iteration 1230/ 2384 | consumed samples: 39360 | elapsed time per iteration (ms): 243431.6 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.57% | FLOPs estimator: workload-aware | logical tokens/s: 68919.5 | valid tokens/s: 68919.5 | physical tokens/s: 69576.1 | loss tokens/s: 45641.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.696827E-01 | lm-global-supervised-tokens: 1.111053E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.584997E-02 | lm-token-buffer-mib: 2.326959E+02 | global_load_balancing_loss: 1.001185E+00 | loss scale: 1.0 | grad norm: 0.116 | params norm: 11178.250 | params rms: 0.060527 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:02:00.045048 | finish at 2026-10-18 01:15:36 + [2026-10-14 19:17:20] iteration 1231/ 2384 | consumed samples: 39392 | elapsed time per iteration (ms): 223724.4 | throughput per GPU (TFLOP/s/GPU): 23.8 | MFU 7.61% | FLOPs estimator: workload-aware | logical tokens/s: 74990.4 | valid tokens/s: 74990.4 | physical tokens/s: 75807.0 | loss tokens/s: 58645.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.854593E-01 | lm-global-supervised-tokens: 1.312051E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.480871E-02 | lm-token-buffer-mib: 2.329994E+02 | global_load_balancing_loss: 1.001474E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11178.218 | params rms: 0.060526 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:39:14.212633 | finish at 2026-10-17 18:56:34 + [2026-10-14 19:21:03] iteration 1232/ 2384 | consumed samples: 39424 | elapsed time per iteration (ms): 223239.9 | throughput per GPU (TFLOP/s/GPU): 24.9 | MFU 7.97% | FLOPs estimator: workload-aware | logical tokens/s: 75153.0 | valid tokens/s: 75153.0 | physical tokens/s: 75438.6 | loss tokens/s: 53249.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.848588E-01 | lm-global-supervised-tokens: 1.188732E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.749833E-02 | lm-token-buffer-mib: 2.330574E+02 | global_load_balancing_loss: 1.004338E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11178.187 | params rms: 0.060526 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:26:12.339386 | finish at 2026-10-17 18:47:15 + [2026-10-14 19:24:46] iteration 1233/ 2384 | consumed samples: 39456 | elapsed time per iteration (ms): 223075.1 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.14% | FLOPs estimator: workload-aware | logical tokens/s: 75208.8 | valid tokens/s: 75208.8 | physical tokens/s: 75900.0 | loss tokens/s: 55636.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.963446E-01 | lm-global-supervised-tokens: 1.241121E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.752227E-02 | lm-token-buffer-mib: 2.348080E+02 | global_load_balancing_loss: 1.000557E+00 | loss scale: 1.0 | grad norm: 0.144 | params norm: 11178.155 | params rms: 0.060526 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:19:19.390367 | finish at 2026-10-17 18:44:05 + [2026-10-14 19:28:30] iteration 1234/ 2384 | consumed samples: 39488 | elapsed time per iteration (ms): 223463.5 | throughput per GPU (TFLOP/s/GPU): 24.9 | MFU 7.98% | FLOPs estimator: workload-aware | logical tokens/s: 75077.9 | valid tokens/s: 75077.9 | physical tokens/s: 75567.6 | loss tokens/s: 54725.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.133107E-01 | lm-global-supervised-tokens: 1.222923E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.113830E-02 | lm-token-buffer-mib: 2.343388E+02 | global_load_balancing_loss: 9.994747E-01 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11178.125 | params rms: 0.060526 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:23:02.980335 | finish at 2026-10-17 18:51:33 + [2026-10-14 19:32:11] iteration 1235/ 2384 | consumed samples: 39520 | elapsed time per iteration (ms): 221945.1 | throughput per GPU (TFLOP/s/GPU): 26.2 | MFU 8.39% | FLOPs estimator: workload-aware | logical tokens/s: 75591.4 | valid tokens/s: 75591.4 | physical tokens/s: 76125.5 | loss tokens/s: 54915.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.620901E-01 | lm-global-supervised-tokens: 1.218822E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.176031E-01 | lm-token-buffer-mib: 2.331589E+02 | global_load_balancing_loss: 1.000716E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11178.093 | params rms: 0.060526 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:50:14.866834 | finish at 2026-10-17 18:22:26 + [2026-10-14 19:35:52] iteration 1236/ 2384 | consumed samples: 39552 | elapsed time per iteration (ms): 220382.9 | throughput per GPU (TFLOP/s/GPU): 24.0 | MFU 7.69% | FLOPs estimator: workload-aware | logical tokens/s: 76127.5 | valid tokens/s: 76127.5 | physical tokens/s: 76449.6 | loss tokens/s: 56227.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.051155E-01 | lm-global-supervised-tokens: 1.239160E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.390898E-02 | lm-token-buffer-mib: 2.329289E+02 | global_load_balancing_loss: 9.990638E-01 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11178.059 | params rms: 0.060526 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:16:39.530060 | finish at 2026-10-17 17:52:31 + [2026-10-14 19:39:34] iteration 1237/ 2384 | consumed samples: 39584 | elapsed time per iteration (ms): 222440.5 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.84% | FLOPs estimator: workload-aware | logical tokens/s: 75423.4 | valid tokens/s: 75423.4 | physical tokens/s: 76353.3 | loss tokens/s: 60378.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.987812E-01 | lm-global-supervised-tokens: 1.343059E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.455125E-02 | lm-token-buffer-mib: 2.354688E+02 | global_load_balancing_loss: 1.000260E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11178.025 | params rms: 0.060525 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:52:19.272120 | finish at 2026-10-17 18:31:54 + [2026-10-14 19:43:17] iteration 1238/ 2384 | consumed samples: 39616 | elapsed time per iteration (ms): 222725.2 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.95% | FLOPs estimator: workload-aware | logical tokens/s: 75326.9 | valid tokens/s: 75326.9 | physical tokens/s: 75740.8 | loss tokens/s: 58107.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.609098E-01 | lm-global-supervised-tokens: 1.294199E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.684203E-02 | lm-token-buffer-mib: 2.331014E+02 | global_load_balancing_loss: 1.000948E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11177.993 | params rms: 0.060525 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:54:03.093610 | finish at 2026-10-17 18:37:20 + [2026-10-14 19:46:59] iteration 1239/ 2384 | consumed samples: 39648 | elapsed time per iteration (ms): 222392.5 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.51% | FLOPs estimator: workload-aware | logical tokens/s: 75439.6 | valid tokens/s: 75439.6 | physical tokens/s: 75996.2 | loss tokens/s: 54437.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.123561E-01 | lm-global-supervised-tokens: 1.210657E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.403229E-02 | lm-token-buffer-mib: 2.329165E+02 | global_load_balancing_loss: 1.001381E+00 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11177.961 | params rms: 0.060525 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:43:59.362183 | finish at 2026-10-17 18:30:59 + [2026-10-14 19:51:03] iteration 1240/ 2384 | consumed samples: 39680 | elapsed time per iteration (ms): 243482.1 | throughput per GPU (TFLOP/s/GPU): 17.8 | MFU 5.72% | FLOPs estimator: workload-aware | logical tokens/s: 68905.3 | valid tokens/s: 68905.3 | physical tokens/s: 69784.3 | loss tokens/s: 52829.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.509335E-01 | lm-global-supervised-tokens: 1.286307E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.692900E-02 | lm-token-buffer-mib: 2.353288E+02 | global_load_balancing_loss: 1.002034E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11177.927 | params rms: 0.060525 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:22:23.465406 | finish at 2026-10-18 01:13:26 +times across ranks (ms): + save-checkpoint: + rank 0: 26141.91 + rank 1: 26141.83 + rank 2: 26141.90 + rank 3: 26141.88 + rank 4: 26141.85 + rank 5: 26141.88 + rank 6: 26141.87 + rank 7: 26141.86 + rank 8: 26141.65 + rank 9: 26141.66 + rank 10: 26141.66 + rank 11: 26141.70 + rank 12: 26141.65 + rank 13: 26141.66 + rank 14: 26141.58 + rank 15: 26141.64 + rank 16: 26141.74 + rank 17: 26141.70 + rank 18: 26141.72 + rank 19: 26141.70 + rank 20: 26141.69 + rank 21: 26141.70 + rank 22: 26141.78 + rank 23: 26141.71 + rank 24: 26141.80 + rank 25: 26141.78 + rank 26: 26141.81 + rank 27: 26141.53 + rank 28: 26141.76 + rank 29: 26142.40 + rank 30: 26141.80 + rank 31: 26142.09 + rank 32: 26141.59 + rank 33: 26141.84 + rank 34: 26141.47 + rank 35: 26141.73 + rank 36: 26141.71 + rank 37: 26141.71 + rank 38: 26141.70 + rank 39: 26141.68 + rank 40: 26142.35 + rank 41: 26142.31 + rank 42: 26142.32 + rank 43: 26142.32 + rank 44: 26142.30 + rank 45: 26142.31 + rank 46: 26142.31 + rank 47: 26142.31 + rank 48: 26141.39 + rank 49: 26141.43 + rank 50: 26141.45 + rank 51: 26141.38 + rank 52: 26141.38 + rank 53: 26141.54 + rank 54: 26141.44 + rank 55: 26141.35 + rank 56: 26141.93 + rank 57: 26141.95 + rank 58: 26141.97 + rank 59: 26141.97 + rank 60: 26141.97 + rank 61: 26141.92 + rank 62: 26141.92 + rank 63: 26141.92 + [2026-10-14 19:55:07] iteration 1241/ 2384 | consumed samples: 39712 | elapsed time per iteration (ms): 218027.8 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.03% | FLOPs estimator: workload-aware | logical tokens/s: 76949.9 | valid tokens/s: 76949.9 | physical tokens/s: 77417.2 | loss tokens/s: 57629.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.358747E-01 | lm-global-supervised-tokens: 1.256472E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.446148E-02 | lm-token-buffer-mib: 2.330683E+02 | global_load_balancing_loss: 1.000296E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11177.892 | params rms: 0.060525 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:13:25.778220 | finish at 2026-10-17 17:08:33 + [2026-10-14 19:58:54] iteration 1242/ 2384 | consumed samples: 39744 | elapsed time per iteration (ms): 226317.9 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.94% | FLOPs estimator: workload-aware | logical tokens/s: 74131.0 | valid tokens/s: 74131.0 | physical tokens/s: 74453.8 | loss tokens/s: 55250.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.087453E-01 | lm-global-supervised-tokens: 1.250420E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.396111E-02 | lm-token-buffer-mib: 2.325428E+02 | global_load_balancing_loss: 1.000054E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11177.862 | params rms: 0.060524 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:47:35.091016 | finish at 2026-10-17 19:46:29 + [2026-10-14 20:02:35] iteration 1243/ 2384 | consumed samples: 39776 | elapsed time per iteration (ms): 221190.1 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.82% | FLOPs estimator: workload-aware | logical tokens/s: 75849.7 | valid tokens/s: 75849.7 | physical tokens/s: 76320.2 | loss tokens/s: 57612.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.499428E-01 | lm-global-supervised-tokens: 1.274336E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.363160E-02 | lm-token-buffer-mib: 2.332837E+02 | global_load_balancing_loss: 1.000478E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11177.831 | params rms: 0.060524 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:06:17.904864 | finish at 2026-10-17 18:08:53 + [2026-10-14 20:06:20] iteration 1244/ 2384 | consumed samples: 39808 | elapsed time per iteration (ms): 225063.4 | throughput per GPU (TFLOP/s/GPU): 24.2 | MFU 7.75% | FLOPs estimator: workload-aware | logical tokens/s: 74544.3 | valid tokens/s: 74544.3 | physical tokens/s: 75228.3 | loss tokens/s: 59029.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.474210E-01 | lm-global-supervised-tokens: 1.328527E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.458628E-02 | lm-token-buffer-mib: 2.356615E+02 | global_load_balancing_loss: 9.990667E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11177.799 | params rms: 0.060524 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:16:12.328491 | finish at 2026-10-17 19:22:32 + [2026-10-14 20:10:01] iteration 1245/ 2384 | consumed samples: 39840 | elapsed time per iteration (ms): 221311.8 | throughput per GPU (TFLOP/s/GPU): 24.3 | MFU 7.77% | FLOPs estimator: workload-aware | logical tokens/s: 75807.9 | valid tokens/s: 75807.9 | physical tokens/s: 76756.3 | loss tokens/s: 55343.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.787077E-01 | lm-global-supervised-tokens: 1.224811E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.335188E-02 | lm-token-buffer-mib: 2.355150E+02 | global_load_balancing_loss: 1.000114E+00 | loss scale: 1.0 | grad norm: 0.108 | params norm: 11177.765 | params rms: 0.060524 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:01:14.093495 | finish at 2026-10-17 18:11:15 + [2026-10-14 20:13:45] iteration 1246/ 2384 | consumed samples: 39872 | elapsed time per iteration (ms): 223615.1 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.97% | FLOPs estimator: workload-aware | logical tokens/s: 75027.2 | valid tokens/s: 75027.2 | physical tokens/s: 75800.3 | loss tokens/s: 53359.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.587502E-01 | lm-global-supervised-tokens: 1.193189E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.019358E-02 | lm-token-buffer-mib: 2.348928E+02 | global_load_balancing_loss: 1.000589E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11177.729 | params rms: 0.060524 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:41:13.931601 | finish at 2026-10-17 18:54:59 + [2026-10-14 20:17:30] iteration 1247/ 2384 | consumed samples: 39904 | elapsed time per iteration (ms): 224663.9 | throughput per GPU (TFLOP/s/GPU): 23.2 | MFU 7.44% | FLOPs estimator: workload-aware | logical tokens/s: 74676.8 | valid tokens/s: 74676.8 | physical tokens/s: 75173.5 | loss tokens/s: 55686.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.872892E-01 | lm-global-supervised-tokens: 1.251072E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.287392E-02 | lm-token-buffer-mib: 2.341467E+02 | global_load_balancing_loss: 1.001312E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11177.693 | params rms: 0.060524 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:57:22.875329 | finish at 2026-10-17 19:14:52 + [2026-10-14 20:21:10] iteration 1248/ 2384 | consumed samples: 39936 | elapsed time per iteration (ms): 220699.3 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.84% | FLOPs estimator: workload-aware | logical tokens/s: 76018.4 | valid tokens/s: 76018.4 | physical tokens/s: 76742.2 | loss tokens/s: 56115.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.658384E-01 | lm-global-supervised-tokens: 1.238457E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.037838E-02 | lm-token-buffer-mib: 2.337769E+02 | global_load_balancing_loss: 1.002959E+00 | loss scale: 1.0 | grad norm: 0.171 | params norm: 11177.660 | params rms: 0.060523 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:38:34.418671 | finish at 2026-10-17 17:59:45 + [2026-10-14 20:24:50] iteration 1249/ 2384 | consumed samples: 39968 | elapsed time per iteration (ms): 219737.9 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.56% | FLOPs estimator: workload-aware | logical tokens/s: 76351.0 | valid tokens/s: 76351.0 | physical tokens/s: 76676.1 | loss tokens/s: 55845.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.318657E-01 | lm-global-supervised-tokens: 1.227144E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.809693E-02 | lm-token-buffer-mib: 2.326753E+02 | global_load_balancing_loss: 1.001042E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11177.625 | params rms: 0.060523 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:16:42.543041 | finish at 2026-10-17 17:41:33 + [2026-10-14 20:28:54] iteration 1250/ 2384 | consumed samples: 40000 | elapsed time per iteration (ms): 243568.1 | throughput per GPU (TFLOP/s/GPU): 19.0 | MFU 6.10% | FLOPs estimator: workload-aware | logical tokens/s: 68880.9 | valid tokens/s: 68880.9 | physical tokens/s: 69826.2 | loss tokens/s: 47483.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.887275E-01 | lm-global-supervised-tokens: 1.156534E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.429311E-02 | lm-token-buffer-mib: 2.358051E+02 | global_load_balancing_loss: 1.000083E+00 | loss scale: 1.0 | grad norm: 0.341 | params norm: 11177.590 | params rms: 0.060523 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:43:26.194821 | finish at 2026-10-18 01:12:20 + [2026-10-14 20:32:36] iteration 1251/ 2384 | consumed samples: 40032 | elapsed time per iteration (ms): 222069.5 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.76% | FLOPs estimator: workload-aware | logical tokens/s: 75549.3 | valid tokens/s: 75549.3 | physical tokens/s: 76304.5 | loss tokens/s: 59326.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.810990E-01 | lm-global-supervised-tokens: 1.317452E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.858446E-02 | lm-token-buffer-mib: 2.347111E+02 | global_load_balancing_loss: 9.988320E-01 | loss scale: 1.0 | grad norm: 0.191 | params norm: 11177.554 | params rms: 0.060523 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:53:24.799652 | finish at 2026-10-17 18:26:00 + [2026-10-14 20:36:20] iteration 1252/ 2384 | consumed samples: 40064 | elapsed time per iteration (ms): 223855.4 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.01% | FLOPs estimator: workload-aware | logical tokens/s: 74946.5 | valid tokens/s: 74946.5 | physical tokens/s: 75541.9 | loss tokens/s: 58203.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.471251E-01 | lm-global-supervised-tokens: 1.302913E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.556551E-02 | lm-token-buffer-mib: 2.350184E+02 | global_load_balancing_loss: 9.981052E-01 | loss scale: 1.0 | grad norm: 0.214 | params norm: 11177.520 | params rms: 0.060523 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:23:24.275112 | finish at 2026-10-17 18:59:44 + [2026-10-14 20:40:00] iteration 1253/ 2384 | consumed samples: 40096 | elapsed time per iteration (ms): 220437.5 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.09% | FLOPs estimator: workload-aware | logical tokens/s: 76108.6 | valid tokens/s: 76108.6 | physical tokens/s: 76531.2 | loss tokens/s: 54439.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.878500E-01 | lm-global-supervised-tokens: 1.200045E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.313286E-02 | lm-token-buffer-mib: 2.334520E+02 | global_load_balancing_loss: 9.987649E-01 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11177.488 | params rms: 0.060522 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:15:14.800905 | finish at 2026-10-17 17:55:15 + [2026-10-14 20:43:44] iteration 1254/ 2384 | consumed samples: 40128 | elapsed time per iteration (ms): 224216.0 | throughput per GPU (TFLOP/s/GPU): 19.6 | MFU 6.27% | FLOPs estimator: workload-aware | logical tokens/s: 74826.0 | valid tokens/s: 74826.0 | physical tokens/s: 75706.1 | loss tokens/s: 56638.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.902056E-01 | lm-global-supervised-tokens: 1.269930E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.255784E-02 | lm-token-buffer-mib: 2.354034E+02 | global_load_balancing_loss: 9.998629E-01 | loss scale: 1.0 | grad norm: 0.229 | params norm: 11177.454 | params rms: 0.060522 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:22:44.108918 | finish at 2026-10-17 19:06:28 + [2026-10-14 20:47:29] iteration 1255/ 2384 | consumed samples: 40160 | elapsed time per iteration (ms): 224519.7 | throughput per GPU (TFLOP/s/GPU): 18.8 | MFU 6.03% | FLOPs estimator: workload-aware | logical tokens/s: 74724.9 | valid tokens/s: 74724.9 | physical tokens/s: 75240.6 | loss tokens/s: 56192.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.109002E-01 | lm-global-supervised-tokens: 1.261624E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.332736E-02 | lm-token-buffer-mib: 2.334363E+02 | global_load_balancing_loss: 1.000342E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11177.420 | params rms: 0.060522 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:24:42.744856 | finish at 2026-10-17 19:12:11 + [2026-10-14 20:51:13] iteration 1256/ 2384 | consumed samples: 40192 | elapsed time per iteration (ms): 224676.4 | throughput per GPU (TFLOP/s/GPU): 25.1 | MFU 8.06% | FLOPs estimator: workload-aware | logical tokens/s: 74672.7 | valid tokens/s: 74672.7 | physical tokens/s: 75244.5 | loss tokens/s: 59505.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.867020E-01 | lm-global-supervised-tokens: 1.336939E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.612704E-02 | lm-token-buffer-mib: 2.339422E+02 | global_load_balancing_loss: 1.001910E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11177.386 | params rms: 0.060522 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:23:54.981560 | finish at 2026-10-17 19:15:08 + [2026-10-14 20:54:55] iteration 1257/ 2384 | consumed samples: 40224 | elapsed time per iteration (ms): 221870.9 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.70% | FLOPs estimator: workload-aware | logical tokens/s: 75616.9 | valid tokens/s: 75616.9 | physical tokens/s: 76300.7 | loss tokens/s: 57198.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.144432E-01 | lm-global-supervised-tokens: 1.269064E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.750879E-02 | lm-token-buffer-mib: 2.340031E+02 | global_load_balancing_loss: 1.001089E+00 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11177.350 | params rms: 0.060522 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:27:28.494801 | finish at 2026-10-17 18:22:24 + [2026-10-14 20:58:36] iteration 1258/ 2384 | consumed samples: 40256 | elapsed time per iteration (ms): 220979.0 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.62% | FLOPs estimator: workload-aware | logical tokens/s: 75922.1 | valid tokens/s: 75922.1 | physical tokens/s: 76438.3 | loss tokens/s: 55943.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.437981E-01 | lm-global-supervised-tokens: 1.236233E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.354886E-02 | lm-token-buffer-mib: 2.324836E+02 | global_load_balancing_loss: 1.002437E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11177.316 | params rms: 0.060522 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:07:02.347660 | finish at 2026-10-17 18:05:39 + [2026-10-14 21:02:21] iteration 1259/ 2384 | consumed samples: 40288 | elapsed time per iteration (ms): 224659.0 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.91% | FLOPs estimator: workload-aware | logical tokens/s: 74678.5 | valid tokens/s: 74678.5 | physical tokens/s: 75280.5 | loss tokens/s: 57292.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.851910E-01 | lm-global-supervised-tokens: 1.287126E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.417383E-02 | lm-token-buffer-mib: 2.349109E+02 | global_load_balancing_loss: 1.001100E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11177.285 | params rms: 0.060521 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:12:21.321266 | finish at 2026-10-17 19:14:42 + [2026-10-14 21:06:27] iteration 1260/ 2384 | consumed samples: 40320 | elapsed time per iteration (ms): 246110.4 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.97% | FLOPs estimator: workload-aware | logical tokens/s: 68169.2 | valid tokens/s: 68169.2 | physical tokens/s: 69045.3 | loss tokens/s: 51833.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.322473E-01 | lm-global-supervised-tokens: 1.275684E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.705777E-02 | lm-token-buffer-mib: 2.360014E+02 | global_load_balancing_loss: 1.001807E+00 | loss scale: 1.0 | grad norm: 0.131 | params norm: 11177.251 | params rms: 0.060521 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:50:28.129754 | finish at 2026-10-18 01:56:55 + [2026-10-14 21:10:14] iteration 1261/ 2384 | consumed samples: 40352 | elapsed time per iteration (ms): 226604.7 | throughput per GPU (TFLOP/s/GPU): 18.9 | MFU 6.07% | FLOPs estimator: workload-aware | logical tokens/s: 74037.3 | valid tokens/s: 74037.3 | physical tokens/s: 75357.2 | loss tokens/s: 51986.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.267999E-01 | lm-global-supervised-tokens: 1.178048E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.718702E-02 | lm-token-buffer-mib: 2.383908E+02 | global_load_balancing_loss: 9.988410E-01 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11177.218 | params rms: 0.060521 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:41:17.070167 | finish at 2026-10-17 19:51:31 + [2026-10-14 21:13:58] iteration 1262/ 2384 | consumed samples: 40384 | elapsed time per iteration (ms): 224066.8 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.77% | FLOPs estimator: workload-aware | logical tokens/s: 74875.9 | valid tokens/s: 74875.9 | physical tokens/s: 75681.4 | loss tokens/s: 56206.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.970893E-01 | lm-global-supervised-tokens: 1.259399E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.031410E-02 | lm-token-buffer-mib: 2.356028E+02 | global_load_balancing_loss: 9.985445E-01 | loss scale: 1.0 | grad norm: 0.112 | params norm: 11177.184 | params rms: 0.060521 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:50:02.957489 | finish at 2026-10-17 19:04:01 + [2026-10-14 21:17:41] iteration 1263/ 2384 | consumed samples: 40416 | elapsed time per iteration (ms): 223552.5 | throughput per GPU (TFLOP/s/GPU): 19.7 | MFU 6.32% | FLOPs estimator: workload-aware | logical tokens/s: 75048.1 | valid tokens/s: 75048.1 | physical tokens/s: 75877.3 | loss tokens/s: 58109.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.482622E-01 | lm-global-supervised-tokens: 1.299046E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.567968E-02 | lm-token-buffer-mib: 2.366907E+02 | global_load_balancing_loss: 9.984522E-01 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11177.151 | params rms: 0.060521 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:36:42.359192 | finish at 2026-10-17 18:54:24 + [2026-10-14 21:21:26] iteration 1264/ 2384 | consumed samples: 40448 | elapsed time per iteration (ms): 224270.7 | throughput per GPU (TFLOP/s/GPU): 25.1 | MFU 8.05% | FLOPs estimator: workload-aware | logical tokens/s: 74807.8 | valid tokens/s: 74807.8 | physical tokens/s: 75841.5 | loss tokens/s: 54568.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.289764E-01 | lm-global-supervised-tokens: 1.223801E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.364700E-02 | lm-token-buffer-mib: 2.339952E+02 | global_load_balancing_loss: 1.002698E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11177.120 | params rms: 0.060520 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:46:23.190117 | finish at 2026-10-17 19:07:49 + [2026-10-14 21:25:09] iteration 1265/ 2384 | consumed samples: 40480 | elapsed time per iteration (ms): 223467.9 | throughput per GPU (TFLOP/s/GPU): 23.9 | MFU 7.67% | FLOPs estimator: workload-aware | logical tokens/s: 75076.5 | valid tokens/s: 75076.5 | physical tokens/s: 75501.4 | loss tokens/s: 55685.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.237546E-01 | lm-global-supervised-tokens: 1.244399E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.243407E-02 | lm-token-buffer-mib: 2.329034E+02 | global_load_balancing_loss: 1.000365E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11177.088 | params rms: 0.060520 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:27:40.534521 | finish at 2026-10-17 18:52:50 + [2026-10-14 21:28:49] iteration 1266/ 2384 | consumed samples: 40512 | elapsed time per iteration (ms): 220200.7 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.98% | FLOPs estimator: workload-aware | logical tokens/s: 76190.5 | valid tokens/s: 76190.5 | physical tokens/s: 76739.6 | loss tokens/s: 56120.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.862049E-01 | lm-global-supervised-tokens: 1.235786E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.173953E-02 | lm-token-buffer-mib: 2.333843E+02 | global_load_balancing_loss: 1.001253E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11177.050 | params rms: 0.060520 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:23:04.343467 | finish at 2026-10-17 17:51:54 + [2026-10-14 21:32:34] iteration 1267/ 2384 | consumed samples: 40544 | elapsed time per iteration (ms): 224623.8 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.75% | FLOPs estimator: workload-aware | logical tokens/s: 74690.2 | valid tokens/s: 74690.2 | physical tokens/s: 75228.2 | loss tokens/s: 52957.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.517867E-01 | lm-global-supervised-tokens: 1.189560E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.236799E-02 | lm-token-buffer-mib: 2.338669E+02 | global_load_balancing_loss: 9.999951E-01 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11177.007 | params rms: 0.060520 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:41:44.838439 | finish at 2026-10-17 19:14:19 + [2026-10-14 21:36:16] iteration 1268/ 2384 | consumed samples: 40576 | elapsed time per iteration (ms): 222008.6 | throughput per GPU (TFLOP/s/GPU): 25.2 | MFU 8.07% | FLOPs estimator: workload-aware | logical tokens/s: 75569.9 | valid tokens/s: 75569.9 | physical tokens/s: 75831.6 | loss tokens/s: 58193.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.701863E-01 | lm-global-supervised-tokens: 1.291938E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.000175E-02 | lm-token-buffer-mib: 2.324527E+02 | global_load_balancing_loss: 1.002496E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11176.964 | params rms: 0.060520 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:49:21.608771 | finish at 2026-10-17 18:25:37 + [2026-10-14 21:40:05] iteration 1269/ 2384 | consumed samples: 40608 | elapsed time per iteration (ms): 228954.7 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.18% | FLOPs estimator: workload-aware | logical tokens/s: 73277.3 | valid tokens/s: 73277.3 | physical tokens/s: 73569.0 | loss tokens/s: 56908.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.412135E-01 | lm-global-supervised-tokens: 1.302936E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.327611E-02 | lm-token-buffer-mib: 2.321625E+02 | global_load_balancing_loss: 1.001800E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11176.924 | params rms: 0.060519 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:54:44.513620 | finish at 2026-10-17 20:34:49 + [2026-10-14 21:44:09] iteration 1270/ 2384 | consumed samples: 40640 | elapsed time per iteration (ms): 244646.2 | throughput per GPU (TFLOP/s/GPU): 19.9 | MFU 6.37% | FLOPs estimator: workload-aware | logical tokens/s: 68577.3 | valid tokens/s: 68577.3 | physical tokens/s: 69429.2 | loss tokens/s: 50448.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.955661E-01 | lm-global-supervised-tokens: 1.234205E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.777480E-02 | lm-token-buffer-mib: 2.335748E+02 | global_load_balancing_loss: 9.989657E-01 | loss scale: 1.0 | grad norm: 0.119 | params norm: 11176.886 | params rms: 0.060519 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:42:15.873109 | finish at 2026-10-18 01:26:25 + [2026-10-14 21:47:54] iteration 1271/ 2384 | consumed samples: 40672 | elapsed time per iteration (ms): 224924.5 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.22% | FLOPs estimator: workload-aware | logical tokens/s: 74590.2 | valid tokens/s: 74590.2 | physical tokens/s: 75242.6 | loss tokens/s: 55585.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.573059E-01 | lm-global-supervised-tokens: 1.250245E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.620357E-02 | lm-token-buffer-mib: 2.350149E+02 | global_load_balancing_loss: 9.998927E-01 | loss scale: 1.0 | grad norm: 0.103 | params norm: 11176.847 | params rms: 0.060519 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:32:20.987062 | finish at 2026-10-17 19:20:15 + [2026-10-14 21:51:35] iteration 1272/ 2384 | consumed samples: 40704 | elapsed time per iteration (ms): 220256.0 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.95% | FLOPs estimator: workload-aware | logical tokens/s: 76171.4 | valid tokens/s: 76171.4 | physical tokens/s: 76643.6 | loss tokens/s: 58476.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.949858E-01 | lm-global-supervised-tokens: 1.287977E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.610703E-02 | lm-token-buffer-mib: 2.331684E+02 | global_load_balancing_loss: 1.000364E+00 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11176.813 | params rms: 0.060519 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:02:04.659321 | finish at 2026-10-17 17:53:39 + [2026-10-14 21:55:15] iteration 1273/ 2384 | consumed samples: 40736 | elapsed time per iteration (ms): 220771.0 | throughput per GPU (TFLOP/s/GPU): 24.1 | MFU 7.73% | FLOPs estimator: workload-aware | logical tokens/s: 75993.5 | valid tokens/s: 75993.5 | physical tokens/s: 76568.6 | loss tokens/s: 51404.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.598758E-01 | lm-global-supervised-tokens: 1.134858E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.325047E-02 | lm-token-buffer-mib: 2.323021E+02 | global_load_balancing_loss: 9.996255E-01 | loss scale: 1.0 | grad norm: 0.184 | params norm: 11176.780 | params rms: 0.060519 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:07:56.571627 | finish at 2026-10-17 18:03:12 + [2026-10-14 21:58:56] iteration 1274/ 2384 | consumed samples: 40768 | elapsed time per iteration (ms): 220442.0 | throughput per GPU (TFLOP/s/GPU): 19.4 | MFU 6.22% | FLOPs estimator: workload-aware | logical tokens/s: 76107.1 | valid tokens/s: 76107.1 | physical tokens/s: 76728.8 | loss tokens/s: 57706.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.324545E-01 | lm-global-supervised-tokens: 1.272088E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.696677E-02 | lm-token-buffer-mib: 2.343628E+02 | global_load_balancing_loss: 1.002103E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11176.746 | params rms: 0.060518 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:58:10.568826 | finish at 2026-10-17 17:57:06 + [2026-10-14 22:02:40] iteration 1275/ 2384 | consumed samples: 40800 | elapsed time per iteration (ms): 224598.5 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.09% | FLOPs estimator: workload-aware | logical tokens/s: 74698.5 | valid tokens/s: 74698.5 | physical tokens/s: 75860.7 | loss tokens/s: 57933.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.766239E-01 | lm-global-supervised-tokens: 1.301170E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.666597E-02 | lm-token-buffer-mib: 2.359669E+02 | global_load_balancing_loss: 1.001949E+00 | loss scale: 1.0 | grad norm: 0.176 | params norm: 11176.713 | params rms: 0.060518 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:11:19.700556 | finish at 2026-10-17 19:14:00 + [2026-10-14 22:06:27] iteration 1276/ 2384 | consumed samples: 40832 | elapsed time per iteration (ms): 226477.2 | throughput per GPU (TFLOP/s/GPU): 24.3 | MFU 7.77% | FLOPs estimator: workload-aware | logical tokens/s: 74079.0 | valid tokens/s: 74079.0 | physical tokens/s: 74562.8 | loss tokens/s: 58804.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.737236E-01 | lm-global-supervised-tokens: 1.331795E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.256543E-02 | lm-token-buffer-mib: 2.344532E+02 | global_load_balancing_loss: 1.001161E+00 | loss scale: 1.0 | grad norm: 0.113 | params norm: 11176.683 | params rms: 0.060518 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:42:16.755862 | finish at 2026-10-17 19:48:44 + [2026-10-14 22:10:09] iteration 1277/ 2384 | consumed samples: 40864 | elapsed time per iteration (ms): 222004.9 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.65% | FLOPs estimator: workload-aware | logical tokens/s: 75571.3 | valid tokens/s: 75571.3 | physical tokens/s: 76058.3 | loss tokens/s: 55390.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.446925E-01 | lm-global-supervised-tokens: 1.229702E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.486533E-02 | lm-token-buffer-mib: 2.339495E+02 | global_load_balancing_loss: 1.000619E+00 | loss scale: 1.0 | grad norm: 0.131 | params norm: 11176.650 | params rms: 0.060518 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:15:59.393397 | finish at 2026-10-17 18:26:08 + [2026-10-14 22:13:50] iteration 1278/ 2384 | consumed samples: 40896 | elapsed time per iteration (ms): 221480.4 | throughput per GPU (TFLOP/s/GPU): 19.6 | MFU 6.29% | FLOPs estimator: workload-aware | logical tokens/s: 75750.3 | valid tokens/s: 75750.3 | physical tokens/s: 77282.2 | loss tokens/s: 56928.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.965771E-01 | lm-global-supervised-tokens: 1.260856E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.999063E-02 | lm-token-buffer-mib: 2.362835E+02 | global_load_balancing_loss: 1.000029E+00 | loss scale: 1.0 | grad norm: 0.146 | params norm: 11176.620 | params rms: 0.060518 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:02:37.273184 | finish at 2026-10-17 18:16:28 + [2026-10-14 22:17:33] iteration 1279/ 2384 | consumed samples: 40928 | elapsed time per iteration (ms): 222611.0 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.49% | FLOPs estimator: workload-aware | logical tokens/s: 75365.4 | valid tokens/s: 75365.4 | physical tokens/s: 76296.8 | loss tokens/s: 57355.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.972287E-01 | lm-global-supervised-tokens: 1.276791E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.412670E-02 | lm-token-buffer-mib: 2.356975E+02 | global_load_balancing_loss: 1.001730E+00 | loss scale: 1.0 | grad norm: 0.508 | params norm: 11176.589 | params rms: 0.060518 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:19:45.141631 | finish at 2026-10-17 18:37:18 + [2026-10-14 22:21:35] iteration 1280/ 2384 | consumed samples: 40960 | elapsed time per iteration (ms): 241762.2 | throughput per GPU (TFLOP/s/GPU): 19.7 | MFU 6.33% | FLOPs estimator: workload-aware | logical tokens/s: 69395.3 | valid tokens/s: 69395.3 | physical tokens/s: 69890.0 | loss tokens/s: 53387.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.222549E-01 | lm-global-supervised-tokens: 1.290710E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.604214E-02 | lm-token-buffer-mib: 2.338498E+02 | global_load_balancing_loss: 1.000647E+00 | loss scale: 1.0 | grad norm: 0.197 | params norm: 11176.557 | params rms: 0.060517 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:08:25.514729 | finish at 2026-10-18 00:30:00 +times across ranks (ms): + save-checkpoint: + rank 0: 24842.32 + rank 1: 24842.32 + rank 2: 24842.31 + rank 3: 24842.32 + rank 4: 24842.30 + rank 5: 24842.31 + rank 6: 24842.35 + rank 7: 24842.29 + rank 8: 24842.96 + rank 9: 24842.97 + rank 10: 24842.96 + rank 11: 24842.96 + rank 12: 24842.94 + rank 13: 24842.95 + rank 14: 24842.95 + rank 15: 24842.97 + rank 16: 24842.99 + rank 17: 24842.99 + rank 18: 24843.00 + rank 19: 24842.98 + rank 20: 24842.96 + rank 21: 24843.02 + rank 22: 24842.87 + rank 23: 24842.97 + rank 24: 24842.69 + rank 25: 24842.67 + rank 26: 24842.74 + rank 27: 24843.45 + rank 28: 24842.67 + rank 29: 24842.69 + rank 30: 24842.76 + rank 31: 24842.56 + rank 32: 24842.44 + rank 33: 24842.17 + rank 34: 24842.06 + rank 35: 24842.47 + rank 36: 24842.18 + rank 37: 24842.20 + rank 38: 24842.15 + rank 39: 24842.19 + rank 40: 24842.33 + rank 41: 24842.32 + rank 42: 24842.31 + rank 43: 24842.32 + rank 44: 24842.30 + rank 45: 24842.30 + rank 46: 24842.34 + rank 47: 24842.30 + rank 48: 24841.77 + rank 49: 24841.81 + rank 50: 24841.83 + rank 51: 24841.79 + rank 52: 24841.79 + rank 53: 24841.81 + rank 54: 24841.76 + rank 55: 24841.79 + rank 56: 24842.76 + rank 57: 24842.79 + rank 58: 24842.79 + rank 59: 24842.81 + rank 60: 24842.79 + rank 61: 24842.75 + rank 62: 24842.76 + rank 63: 24842.75 + [2026-10-14 22:26:01] iteration 1281/ 2384 | consumed samples: 40992 | elapsed time per iteration (ms): 241243.9 | throughput per GPU (TFLOP/s/GPU): 19.9 | MFU 6.38% | FLOPs estimator: workload-aware | logical tokens/s: 69544.5 | valid tokens/s: 69544.5 | physical tokens/s: 70731.0 | loss tokens/s: 52902.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.015213E-01 | lm-global-supervised-tokens: 1.276228E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.094176E+00 | lm-token-buffer-mib: 2.330952E+02 | global_load_balancing_loss: 9.995441E-01 | loss scale: 1.0 | grad norm: 0.337 | params norm: 11176.523 | params rms: 0.060517 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:54:52.036494 | finish at 2026-10-18 00:20:53 + [2026-10-14 22:29:54] iteration 1282/ 2384 | consumed samples: 41024 | elapsed time per iteration (ms): 232923.6 | throughput per GPU (TFLOP/s/GPU): 19.1 | MFU 6.11% | FLOPs estimator: workload-aware | logical tokens/s: 72028.8 | valid tokens/s: 72028.8 | physical tokens/s: 72718.0 | loss tokens/s: 53630.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.776293E-01 | lm-global-supervised-tokens: 1.249170E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.890841E-02 | lm-token-buffer-mib: 2.356841E+02 | global_load_balancing_loss: 1.001030E+00 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11176.493 | params rms: 0.060517 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:18:01.826859 | finish at 2026-10-17 21:47:56 + [2026-10-14 22:33:36] iteration 1283/ 2384 | consumed samples: 41056 | elapsed time per iteration (ms): 222201.6 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.19% | FLOPs estimator: workload-aware | logical tokens/s: 75504.3 | valid tokens/s: 75504.3 | physical tokens/s: 76252.8 | loss tokens/s: 58939.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.730556E-01 | lm-global-supervised-tokens: 1.309649E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.594632E-02 | lm-token-buffer-mib: 2.345481E+02 | global_load_balancing_loss: 9.990497E-01 | loss scale: 1.0 | grad norm: 0.422 | params norm: 11176.462 | params rms: 0.060517 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:57:23.988457 | finish at 2026-10-17 18:31:00 + [2026-10-14 22:37:18] iteration 1284/ 2384 | consumed samples: 41088 | elapsed time per iteration (ms): 221863.9 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.47% | FLOPs estimator: workload-aware | logical tokens/s: 75619.4 | valid tokens/s: 75619.4 | physical tokens/s: 76040.3 | loss tokens/s: 56562.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.032342E-01 | lm-global-supervised-tokens: 1.254924E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.546066E-02 | lm-token-buffer-mib: 2.342097E+02 | global_load_balancing_loss: 9.990722E-01 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11176.430 | params rms: 0.060517 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:47:30.241423 | finish at 2026-10-17 18:24:48 + [2026-10-14 22:41:00] iteration 1285/ 2384 | consumed samples: 41120 | elapsed time per iteration (ms): 221328.9 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.55% | FLOPs estimator: workload-aware | logical tokens/s: 75802.1 | valid tokens/s: 75802.1 | physical tokens/s: 75955.2 | loss tokens/s: 54321.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.528326E-01 | lm-global-supervised-tokens: 1.202283E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.129630E-02 | lm-token-buffer-mib: 2.325312E+02 | global_load_balancing_loss: 1.000122E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11176.392 | params rms: 0.060517 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:34:00.449156 | finish at 2026-10-17 18:15:00 + [2026-10-14 22:44:41] iteration 1286/ 2384 | consumed samples: 41152 | elapsed time per iteration (ms): 221597.6 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.36% | FLOPs estimator: workload-aware | logical tokens/s: 75710.1 | valid tokens/s: 75710.1 | physical tokens/s: 76212.2 | loss tokens/s: 58126.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.856153E-01 | lm-global-supervised-tokens: 1.288065E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.711792E-02 | lm-token-buffer-mib: 2.338337E+02 | global_load_balancing_loss: 1.000085E+00 | loss scale: 1.0 | grad norm: 0.193 | params norm: 11176.356 | params rms: 0.060516 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:35:14.171326 | finish at 2026-10-17 18:19:55 + [2026-10-14 22:48:25] iteration 1287/ 2384 | consumed samples: 41184 | elapsed time per iteration (ms): 223585.1 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.02% | FLOPs estimator: workload-aware | logical tokens/s: 75037.1 | valid tokens/s: 75037.1 | physical tokens/s: 76157.3 | loss tokens/s: 55588.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.296380E-01 | lm-global-supervised-tokens: 1.242868E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.361084E-02 | lm-token-buffer-mib: 2.362611E+02 | global_load_balancing_loss: 1.002168E+00 | loss scale: 1.0 | grad norm: 0.268 | params norm: 11176.318 | params rms: 0.060516 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:07:52.891246 | finish at 2026-10-17 18:56:18 + [2026-10-14 22:52:06] iteration 1288/ 2384 | consumed samples: 41216 | elapsed time per iteration (ms): 220980.7 | throughput per GPU (TFLOP/s/GPU): 19.6 | MFU 6.27% | FLOPs estimator: workload-aware | logical tokens/s: 75921.5 | valid tokens/s: 75921.5 | physical tokens/s: 76388.8 | loss tokens/s: 57779.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.989081E-01 | lm-global-supervised-tokens: 1.276812E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.819980E-02 | lm-token-buffer-mib: 2.325237E+02 | global_load_balancing_loss: 1.002552E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11176.280 | params rms: 0.060516 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:16:34.842253 | finish at 2026-10-17 18:08:41 + [2026-10-14 22:55:50] iteration 1289/ 2384 | consumed samples: 41248 | elapsed time per iteration (ms): 224108.1 | throughput per GPU (TFLOP/s/GPU): 25.6 | MFU 8.22% | FLOPs estimator: workload-aware | logical tokens/s: 74861.9 | valid tokens/s: 74861.9 | physical tokens/s: 75473.0 | loss tokens/s: 56569.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.280863E-01 | lm-global-supervised-tokens: 1.267778E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.053118E-02 | lm-token-buffer-mib: 2.342397E+02 | global_load_balancing_loss: 1.000130E+00 | loss scale: 1.0 | grad norm: 0.204 | params norm: 11176.241 | params rms: 0.060516 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:09:58.401543 | finish at 2026-10-17 19:05:48 + [2026-10-14 22:59:54] iteration 1290/ 2384 | consumed samples: 41280 | elapsed time per iteration (ms): 243722.1 | throughput per GPU (TFLOP/s/GPU): 18.6 | MFU 5.95% | FLOPs estimator: workload-aware | logical tokens/s: 68837.4 | valid tokens/s: 68837.4 | physical tokens/s: 69431.5 | loss tokens/s: 52407.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.954682E-01 | lm-global-supervised-tokens: 1.277287E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.381347E-02 | lm-token-buffer-mib: 2.337972E+02 | global_load_balancing_loss: 1.001739E+00 | loss scale: 1.0 | grad norm: 0.238 | params norm: 11176.201 | params rms: 0.060515 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:03:51.970640 | finish at 2026-10-18 01:03:46 + [2026-10-14 23:03:37] iteration 1291/ 2384 | consumed samples: 41312 | elapsed time per iteration (ms): 223573.8 | throughput per GPU (TFLOP/s/GPU): 26.2 | MFU 8.38% | FLOPs estimator: workload-aware | logical tokens/s: 75040.9 | valid tokens/s: 75040.9 | physical tokens/s: 75604.7 | loss tokens/s: 60033.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.488728E-01 | lm-global-supervised-tokens: 1.342187E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.027796E-01 | lm-token-buffer-mib: 2.344908E+02 | global_load_balancing_loss: 1.002146E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11176.160 | params rms: 0.060515 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:52:46.174179 | finish at 2026-10-17 18:56:23 + [2026-10-14 23:07:19] iteration 1292/ 2384 | consumed samples: 41344 | elapsed time per iteration (ms): 221519.8 | throughput per GPU (TFLOP/s/GPU): 24.7 | MFU 7.91% | FLOPs estimator: workload-aware | logical tokens/s: 75736.7 | valid tokens/s: 75736.7 | physical tokens/s: 76035.3 | loss tokens/s: 58286.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.176927E-01 | lm-global-supervised-tokens: 1.291150E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.909728E-02 | lm-token-buffer-mib: 2.330896E+02 | global_load_balancing_loss: 9.993222E-01 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11176.121 | params rms: 0.060515 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:11:39.569733 | finish at 2026-10-17 18:18:58 + [2026-10-14 23:11:04] iteration 1293/ 2384 | consumed samples: 41376 | elapsed time per iteration (ms): 225780.2 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.92% | FLOPs estimator: workload-aware | logical tokens/s: 74307.7 | valid tokens/s: 74307.7 | physical tokens/s: 74800.7 | loss tokens/s: 54240.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.231477E-01 | lm-global-supervised-tokens: 1.224652E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.400769E-02 | lm-token-buffer-mib: 2.324176E+02 | global_load_balancing_loss: 9.983402E-01 | loss scale: 1.0 | grad norm: 0.177 | params norm: 11176.080 | params rms: 0.060515 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:25:26.172454 | finish at 2026-10-17 19:36:31 + [2026-10-14 23:14:46] iteration 1294/ 2384 | consumed samples: 41408 | elapsed time per iteration (ms): 221618.2 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.93% | FLOPs estimator: workload-aware | logical tokens/s: 75703.1 | valid tokens/s: 75703.1 | physical tokens/s: 76013.7 | loss tokens/s: 59518.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.733422E-01 | lm-global-supervised-tokens: 1.319038E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.927704E-02 | lm-token-buffer-mib: 2.331229E+02 | global_load_balancing_loss: 9.983067E-01 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11176.042 | params rms: 0.060515 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:06:03.798048 | finish at 2026-10-17 18:20:50 + [2026-10-14 23:18:33] iteration 1295/ 2384 | consumed samples: 41440 | elapsed time per iteration (ms): 227229.3 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.94% | FLOPs estimator: workload-aware | logical tokens/s: 73833.8 | valid tokens/s: 73833.8 | physical tokens/s: 74363.7 | loss tokens/s: 53670.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.111051E-01 | lm-global-supervised-tokens: 1.219552E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.575687E-02 | lm-token-buffer-mib: 2.341799E+02 | global_load_balancing_loss: 9.991791E-01 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11176.005 | params rms: 0.060514 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:44:12.700974 | finish at 2026-10-17 20:02:46 + [2026-10-14 23:22:18] iteration 1296/ 2384 | consumed samples: 41472 | elapsed time per iteration (ms): 224773.5 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.93% | FLOPs estimator: workload-aware | logical tokens/s: 74640.5 | valid tokens/s: 74640.5 | physical tokens/s: 75464.6 | loss tokens/s: 57707.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.149099E-01 | lm-global-supervised-tokens: 1.297102E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.622188E-02 | lm-token-buffer-mib: 2.357023E+02 | global_load_balancing_loss: 1.001001E+00 | loss scale: 1.0 | grad norm: 0.188 | params norm: 11175.965 | params rms: 0.060514 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:55:53.522308 | finish at 2026-10-17 19:18:12 + [2026-10-14 23:26:00] iteration 1297/ 2384 | consumed samples: 41504 | elapsed time per iteration (ms): 222314.9 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.10% | FLOPs estimator: workload-aware | logical tokens/s: 75465.8 | valid tokens/s: 75465.8 | physical tokens/s: 76306.6 | loss tokens/s: 54721.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.068601E-01 | lm-global-supervised-tokens: 1.216530E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 8.961512E-02 | lm-token-buffer-mib: 2.329825E+02 | global_load_balancing_loss: 1.001256E+00 | loss scale: 1.0 | grad norm: 0.100 | params norm: 11175.925 | params rms: 0.060514 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:07:36.286366 | finish at 2026-10-17 18:33:37 + [2026-10-14 23:29:39] iteration 1298/ 2384 | consumed samples: 41536 | elapsed time per iteration (ms): 218893.0 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.23% | FLOPs estimator: workload-aware | logical tokens/s: 76645.5 | valid tokens/s: 76645.5 | physical tokens/s: 76957.4 | loss tokens/s: 57337.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.400968E-01 | lm-global-supervised-tokens: 1.255066E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.947629E-02 | lm-token-buffer-mib: 2.325967E+02 | global_load_balancing_loss: 1.001743E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11175.884 | params rms: 0.060514 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 18:01:57.770950 | finish at 2026-10-17 17:31:37 + [2026-10-14 23:33:22] iteration 1299/ 2384 | consumed samples: 41568 | elapsed time per iteration (ms): 222475.4 | throughput per GPU (TFLOP/s/GPU): 27.0 | MFU 8.66% | FLOPs estimator: workload-aware | logical tokens/s: 75411.3 | valid tokens/s: 75411.3 | physical tokens/s: 75822.9 | loss tokens/s: 53303.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.860696E-01 | lm-global-supervised-tokens: 1.185860E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.252704E-02 | lm-token-buffer-mib: 2.335641E+02 | global_load_balancing_loss: 1.000580E+00 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11175.844 | params rms: 0.060514 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:03:05.766027 | finish at 2026-10-17 18:36:28 + [2026-10-14 23:37:22] iteration 1300/ 2384 | consumed samples: 41600 | elapsed time per iteration (ms): 240667.8 | throughput per GPU (TFLOP/s/GPU): 19.4 | MFU 6.21% | FLOPs estimator: workload-aware | logical tokens/s: 69711.0 | valid tokens/s: 69711.0 | physical tokens/s: 70294.5 | loss tokens/s: 52895.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.815898E-01 | lm-global-supervised-tokens: 1.273030E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.844087E-02 | lm-token-buffer-mib: 2.341000E+02 | global_load_balancing_loss: 1.001469E+00 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11175.798 | params rms: 0.060513 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 0:28:03.887650 | finish at 2026-10-18 00:05:26 + [2026-10-14 23:41:04] iteration 1301/ 2384 | consumed samples: 41632 | elapsed time per iteration (ms): 221914.3 | throughput per GPU (TFLOP/s/GPU): 24.4 | MFU 7.83% | FLOPs estimator: workload-aware | logical tokens/s: 75602.0 | valid tokens/s: 75602.0 | physical tokens/s: 75822.6 | loss tokens/s: 50419.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.254995E-01 | lm-global-supervised-tokens: 1.118885E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.458145E-02 | lm-token-buffer-mib: 2.327769E+02 | global_load_balancing_loss: 1.001054E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11175.756 | params rms: 0.060513 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 18:45:33.204162 | finish at 2026-10-17 18:26:38 + [2026-10-14 23:44:47] iteration 1302/ 2384 | consumed samples: 41664 | elapsed time per iteration (ms): 223026.5 | throughput per GPU (TFLOP/s/GPU): 24.7 | MFU 7.91% | FLOPs estimator: workload-aware | logical tokens/s: 75225.1 | valid tokens/s: 75225.1 | physical tokens/s: 75909.3 | loss tokens/s: 54900.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.559610E-01 | lm-global-supervised-tokens: 1.224432E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 6.284562E-02 | lm-token-buffer-mib: 2.350737E+02 | global_load_balancing_loss: 1.000501E+00 | loss scale: 1.0 | grad norm: 0.359 | params norm: 11175.718 | params rms: 0.060513 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:01:54.674533 | finish at 2026-10-17 18:46:42 + [2026-10-14 23:48:30] iteration 1303/ 2384 | consumed samples: 41696 | elapsed time per iteration (ms): 222335.2 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.85% | FLOPs estimator: workload-aware | logical tokens/s: 75459.0 | valid tokens/s: 75459.0 | physical tokens/s: 75998.2 | loss tokens/s: 55037.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.524512E-01 | lm-global-supervised-tokens: 1.223685E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.872783E-02 | lm-token-buffer-mib: 2.327791E+02 | global_load_balancing_loss: 1.000704E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11175.679 | params rms: 0.060513 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 18:45:44.391483 | finish at 2026-10-17 18:34:14 + [2026-10-14 23:52:12] iteration 1304/ 2384 | consumed samples: 41728 | elapsed time per iteration (ms): 221934.3 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.56% | FLOPs estimator: workload-aware | logical tokens/s: 75595.2 | valid tokens/s: 75595.2 | physical tokens/s: 76004.3 | loss tokens/s: 57159.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.759724E-01 | lm-global-supervised-tokens: 1.268568E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.341100E-02 | lm-token-buffer-mib: 2.329281E+02 | global_load_balancing_loss: 9.995574E-01 | loss scale: 1.0 | grad norm: 0.414 | params norm: 11175.638 | params rms: 0.060512 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 18:34:49.012012 | finish at 2026-10-17 18:27:01 + [2026-10-14 23:55:58] iteration 1305/ 2384 | consumed samples: 41760 | elapsed time per iteration (ms): 226314.0 | throughput per GPU (TFLOP/s/GPU): 24.9 | MFU 8.00% | FLOPs estimator: workload-aware | logical tokens/s: 74132.4 | valid tokens/s: 74132.4 | physical tokens/s: 74793.9 | loss tokens/s: 57065.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.421999E-01 | lm-global-supervised-tokens: 1.291464E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.346539E-02 | lm-token-buffer-mib: 2.339203E+02 | global_load_balancing_loss: 1.000914E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11175.599 | params rms: 0.060512 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:49:52.784531 | finish at 2026-10-17 19:45:51 + [2026-10-14 23:59:40] iteration 1306/ 2384 | consumed samples: 41792 | elapsed time per iteration (ms): 221774.0 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.59% | FLOPs estimator: workload-aware | logical tokens/s: 75649.8 | valid tokens/s: 75649.8 | physical tokens/s: 76231.0 | loss tokens/s: 57369.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.854420E-01 | lm-global-supervised-tokens: 1.272299E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.569380E-02 | lm-token-buffer-mib: 2.346671E+02 | global_load_balancing_loss: 9.993302E-01 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11175.564 | params rms: 0.060512 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 18:24:32.355218 | finish at 2026-10-17 18:24:12 + [2026-10-15 00:03:26] iteration 1307/ 2384 | consumed samples: 41824 | elapsed time per iteration (ms): 226046.2 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.35% | FLOPs estimator: workload-aware | logical tokens/s: 74220.2 | valid tokens/s: 74220.2 | physical tokens/s: 75748.9 | loss tokens/s: 58161.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.317218E-01 | lm-global-supervised-tokens: 1.314708E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.504669E-02 | lm-token-buffer-mib: 2.354470E+02 | global_load_balancing_loss: 9.993517E-01 | loss scale: 1.0 | grad norm: 0.325 | params norm: 11175.532 | params rms: 0.060512 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:37:31.739465 | finish at 2026-10-17 19:40:58 + [2026-10-15 00:07:12] iteration 1308/ 2384 | consumed samples: 41856 | elapsed time per iteration (ms): 226264.1 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.26% | FLOPs estimator: workload-aware | logical tokens/s: 74148.5 | valid tokens/s: 74148.5 | physical tokens/s: 74573.9 | loss tokens/s: 54324.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.769445E-01 | lm-global-supervised-tokens: 1.229177E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.415602E-02 | lm-token-buffer-mib: 2.333476E+02 | global_load_balancing_loss: 9.999765E-01 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11175.501 | params rms: 0.060512 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:37:40.211187 | finish at 2026-10-17 19:44:52 + [2026-10-15 00:10:54] iteration 1309/ 2384 | consumed samples: 41888 | elapsed time per iteration (ms): 221738.0 | throughput per GPU (TFLOP/s/GPU): 26.0 | MFU 8.32% | FLOPs estimator: workload-aware | logical tokens/s: 75662.1 | valid tokens/s: 75662.1 | physical tokens/s: 76265.6 | loss tokens/s: 59430.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.188936E-01 | lm-global-supervised-tokens: 1.317805E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.961091E-02 | lm-token-buffer-mib: 2.342985E+02 | global_load_balancing_loss: 9.998565E-01 | loss scale: 1.0 | grad norm: 0.316 | params norm: 11175.468 | params rms: 0.060512 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 18:12:48.302494 | finish at 2026-10-17 18:23:42 + [2026-10-15 00:15:00] iteration 1310/ 2384 | consumed samples: 41920 | elapsed time per iteration (ms): 246027.1 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.69% | FLOPs estimator: workload-aware | logical tokens/s: 68192.4 | valid tokens/s: 68192.4 | physical tokens/s: 68910.5 | loss tokens/s: 52389.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.495626E-01 | lm-global-supervised-tokens: 1.288932E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.130080E-02 | lm-token-buffer-mib: 2.342597E+02 | global_load_balancing_loss: 1.001694E+00 | loss scale: 1.0 | grad norm: 0.148 | params norm: 11175.437 | params rms: 0.060511 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:23:53.071436 | finish at 2026-10-18 01:38:53 + [2026-10-15 00:18:42] iteration 1311/ 2384 | consumed samples: 41952 | elapsed time per iteration (ms): 221719.9 | throughput per GPU (TFLOP/s/GPU): 24.5 | MFU 7.84% | FLOPs estimator: workload-aware | logical tokens/s: 75668.3 | valid tokens/s: 75668.3 | physical tokens/s: 76579.8 | loss tokens/s: 61635.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.863480E-01 | lm-global-supervised-tokens: 1.366588E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.401114E-02 | lm-token-buffer-mib: 2.343807E+02 | global_load_balancing_loss: 1.003242E+00 | loss scale: 1.0 | grad norm: 0.154 | params norm: 11175.406 | params rms: 0.060511 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 18:05:05.472283 | finish at 2026-10-17 18:23:47 + [2026-10-15 00:22:23] iteration 1312/ 2384 | consumed samples: 41984 | elapsed time per iteration (ms): 221588.1 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.41% | FLOPs estimator: workload-aware | logical tokens/s: 75713.5 | valid tokens/s: 75713.5 | physical tokens/s: 76468.2 | loss tokens/s: 57554.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.262135E-01 | lm-global-supervised-tokens: 1.275339E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.374623E-02 | lm-token-buffer-mib: 2.334535E+02 | global_load_balancing_loss: 1.001630E+00 | loss scale: 1.0 | grad norm: 0.284 | params norm: 11175.374 | params rms: 0.060511 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:59:02.483025 | finish at 2026-10-17 18:21:26 + [2026-10-15 00:26:09] iteration 1313/ 2384 | consumed samples: 42016 | elapsed time per iteration (ms): 225652.7 | throughput per GPU (TFLOP/s/GPU): 23.2 | MFU 7.44% | FLOPs estimator: workload-aware | logical tokens/s: 74349.4 | valid tokens/s: 74349.4 | physical tokens/s: 75500.9 | loss tokens/s: 55272.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.122802E-01 | lm-global-supervised-tokens: 1.247230E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 5.148648E-02 | lm-token-buffer-mib: 2.362383E+02 | global_load_balancing_loss: 9.998723E-01 | loss scale: 1.0 | grad norm: 0.118 | params norm: 11175.341 | params rms: 0.060511 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:07:54.065135 | finish at 2026-10-17 19:34:03 + [2026-10-15 00:29:49] iteration 1314/ 2384 | consumed samples: 42048 | elapsed time per iteration (ms): 220245.7 | throughput per GPU (TFLOP/s/GPU): 23.4 | MFU 7.51% | FLOPs estimator: workload-aware | logical tokens/s: 76174.9 | valid tokens/s: 76174.9 | physical tokens/s: 76486.5 | loss tokens/s: 53445.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.507172E-01 | lm-global-supervised-tokens: 1.177115E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.631779E-02 | lm-token-buffer-mib: 2.333920E+02 | global_load_balancing_loss: 9.984787E-01 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11175.305 | params rms: 0.060511 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:27:42.951682 | finish at 2026-10-17 17:57:32 + [2026-10-15 00:33:32] iteration 1315/ 2384 | consumed samples: 42080 | elapsed time per iteration (ms): 222803.6 | throughput per GPU (TFLOP/s/GPU): 23.4 | MFU 7.50% | FLOPs estimator: workload-aware | logical tokens/s: 75300.4 | valid tokens/s: 75300.4 | physical tokens/s: 75918.6 | loss tokens/s: 58043.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.264874E-01 | lm-global-supervised-tokens: 1.293229E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.663664E-02 | lm-token-buffer-mib: 2.332912E+02 | global_load_balancing_loss: 9.992537E-01 | loss scale: 1.0 | grad norm: 0.140 | params norm: 11175.265 | params rms: 0.060510 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 18:09:37.037009 | finish at 2026-10-17 18:43:09 + [2026-10-15 00:37:12] iteration 1316/ 2384 | consumed samples: 42112 | elapsed time per iteration (ms): 220349.1 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.71% | FLOPs estimator: workload-aware | logical tokens/s: 76139.2 | valid tokens/s: 76139.2 | physical tokens/s: 76540.7 | loss tokens/s: 55044.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.650304E-01 | lm-global-supervised-tokens: 1.212891E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.994440E-01 | lm-token-buffer-mib: 2.328863E+02 | global_load_balancing_loss: 1.000372E+00 | loss scale: 1.0 | grad norm: 0.212 | params norm: 11175.225 | params rms: 0.060510 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:22:12.820333 | finish at 2026-10-17 17:59:25 + [2026-10-15 00:40:54] iteration 1317/ 2384 | consumed samples: 42144 | elapsed time per iteration (ms): 221669.9 | throughput per GPU (TFLOP/s/GPU): 20.2 | MFU 6.47% | FLOPs estimator: workload-aware | logical tokens/s: 75685.6 | valid tokens/s: 75685.6 | physical tokens/s: 76725.6 | loss tokens/s: 55426.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.664750E-01 | lm-global-supervised-tokens: 1.228628E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.485227E-02 | lm-token-buffer-mib: 2.337529E+02 | global_load_balancing_loss: 9.997655E-01 | loss scale: 1.0 | grad norm: 0.118 | params norm: 11175.191 | params rms: 0.060510 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:42:01.732612 | finish at 2026-10-17 18:22:56 + [2026-10-15 00:44:37] iteration 1318/ 2384 | consumed samples: 42176 | elapsed time per iteration (ms): 222784.9 | throughput per GPU (TFLOP/s/GPU): 24.2 | MFU 7.75% | FLOPs estimator: workload-aware | logical tokens/s: 75306.5 | valid tokens/s: 75306.5 | physical tokens/s: 76107.4 | loss tokens/s: 61706.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.607452E-01 | lm-global-supervised-tokens: 1.374736E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.737499E-02 | lm-token-buffer-mib: 2.336595E+02 | global_load_balancing_loss: 1.000831E+00 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11175.158 | params rms: 0.060510 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:58:08.728254 | finish at 2026-10-17 18:42:45 + [2026-10-15 00:48:17] iteration 1319/ 2384 | consumed samples: 42208 | elapsed time per iteration (ms): 220414.7 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.72% | FLOPs estimator: workload-aware | logical tokens/s: 76116.5 | valid tokens/s: 76116.4 | physical tokens/s: 76844.5 | loss tokens/s: 60648.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.686843E-01 | lm-global-supervised-tokens: 1.336778E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.728852E-02 | lm-token-buffer-mib: 2.337238E+02 | global_load_balancing_loss: 1.001837E+00 | loss scale: 1.0 | grad norm: 0.155 | params norm: 11175.123 | params rms: 0.060510 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:12:21.703523 | finish at 2026-10-17 18:00:39 + [2026-10-15 00:52:24] iteration 1320/ 2384 | consumed samples: 42240 | elapsed time per iteration (ms): 247221.3 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.87% | FLOPs estimator: workload-aware | logical tokens/s: 67862.9 | valid tokens/s: 67862.9 | physical tokens/s: 68697.0 | loss tokens/s: 52001.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.313896E-01 | lm-global-supervised-tokens: 1.285590E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.338692E-02 | lm-token-buffer-mib: 2.345769E+02 | global_load_balancing_loss: 1.001458E+00 | loss scale: 1.0 | grad norm: 0.192 | params norm: 11175.091 | params rms: 0.060509 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:04:03.425028 | finish at 2026-10-18 01:56:28 +times across ranks (ms): + save-checkpoint: + rank 0: 24430.87 + rank 1: 24430.91 + rank 2: 24430.87 + rank 3: 24430.87 + rank 4: 24430.88 + rank 5: 24430.92 + rank 6: 24430.91 + rank 7: 24430.88 + rank 8: 24430.35 + rank 9: 24430.40 + rank 10: 24430.40 + rank 11: 24430.35 + rank 12: 24430.38 + rank 13: 24430.35 + rank 14: 24430.37 + rank 15: 24430.35 + rank 16: 24430.27 + rank 17: 24430.26 + rank 18: 24430.26 + rank 19: 24430.31 + rank 20: 24430.08 + rank 21: 24430.22 + rank 22: 24430.26 + rank 23: 24430.22 + rank 24: 24430.15 + rank 25: 24430.15 + rank 26: 24430.24 + rank 27: 24430.12 + rank 28: 24430.46 + rank 29: 24430.68 + rank 30: 24430.18 + rank 31: 24430.11 + rank 32: 24430.29 + rank 33: 24430.34 + rank 34: 24430.33 + rank 35: 24430.36 + rank 36: 24430.36 + rank 37: 24430.35 + rank 38: 24430.29 + rank 39: 24430.26 + rank 40: 24430.27 + rank 41: 24430.24 + rank 42: 24430.30 + rank 43: 24430.25 + rank 44: 24430.25 + rank 45: 24430.23 + rank 46: 24430.22 + rank 47: 24430.23 + rank 48: 24430.26 + rank 49: 24430.38 + rank 50: 24430.33 + rank 51: 24430.26 + rank 52: 24430.34 + rank 53: 24430.44 + rank 54: 24430.23 + rank 55: 24430.30 + rank 56: 24430.53 + rank 57: 24430.55 + rank 58: 24430.60 + rank 59: 24430.55 + rank 60: 24430.57 + rank 61: 24430.59 + rank 62: 24430.56 + rank 63: 24430.53 + [2026-10-15 00:56:30] iteration 1321/ 2384 | consumed samples: 42272 | elapsed time per iteration (ms): 221522.7 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.14% | FLOPs estimator: workload-aware | logical tokens/s: 75735.8 | valid tokens/s: 75735.8 | physical tokens/s: 75993.3 | loss tokens/s: 53968.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.377423E-01 | lm-global-supervised-tokens: 1.195515E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 5.467451E-02 | lm-token-buffer-mib: 2.331582E+02 | global_load_balancing_loss: 1.001418E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11175.061 | params rms: 0.060509 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:24:38.682638 | finish at 2026-10-17 18:21:09 + [2026-10-15 01:00:18] iteration 1322/ 2384 | consumed samples: 42304 | elapsed time per iteration (ms): 227465.5 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.09% | FLOPs estimator: workload-aware | logical tokens/s: 73757.1 | valid tokens/s: 73757.1 | physical tokens/s: 74682.0 | loss tokens/s: 55875.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.144879E-01 | lm-global-supervised-tokens: 1.270967E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.618419E-02 | lm-token-buffer-mib: 2.323827E+02 | global_load_balancing_loss: 9.991407E-01 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11175.034 | params rms: 0.060509 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:06:08.314028 | finish at 2026-10-17 20:06:26 + [2026-10-15 01:04:00] iteration 1323/ 2384 | consumed samples: 42336 | elapsed time per iteration (ms): 221894.6 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.17% | FLOPs estimator: workload-aware | logical tokens/s: 75608.8 | valid tokens/s: 75608.8 | physical tokens/s: 76559.6 | loss tokens/s: 56397.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.432318E-01 | lm-global-supervised-tokens: 1.251422E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.721379E-02 | lm-token-buffer-mib: 2.353523E+02 | global_load_balancing_loss: 9.994416E-01 | loss scale: 1.0 | grad norm: 0.171 | params norm: 11175.005 | params rms: 0.060509 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:23:50.213766 | finish at 2026-10-17 18:27:50 + [2026-10-15 01:07:39] iteration 1324/ 2384 | consumed samples: 42368 | elapsed time per iteration (ms): 219658.3 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.21% | FLOPs estimator: workload-aware | logical tokens/s: 76378.5 | valid tokens/s: 76378.5 | physical tokens/s: 77024.4 | loss tokens/s: 60090.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.111025E-01 | lm-global-supervised-tokens: 1.319927E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.200153E-02 | lm-token-buffer-mib: 2.344075E+02 | global_load_balancing_loss: 9.981468E-01 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11174.971 | params rms: 0.060509 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:40:37.782755 | finish at 2026-10-17 17:48:17 + [2026-10-15 01:11:22] iteration 1325/ 2384 | consumed samples: 42400 | elapsed time per iteration (ms): 222044.2 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.95% | FLOPs estimator: workload-aware | logical tokens/s: 75558.0 | valid tokens/s: 75558.0 | physical tokens/s: 76161.0 | loss tokens/s: 56032.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.371923E-01 | lm-global-supervised-tokens: 1.244170E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.732512E-02 | lm-token-buffer-mib: 2.340048E+02 | global_load_balancing_loss: 9.999579E-01 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11174.935 | params rms: 0.060509 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:19:04.760020 | finish at 2026-10-17 18:30:26 + [2026-10-15 01:15:04] iteration 1326/ 2384 | consumed samples: 42432 | elapsed time per iteration (ms): 222398.3 | throughput per GPU (TFLOP/s/GPU): 25.9 | MFU 8.31% | FLOPs estimator: workload-aware | logical tokens/s: 75437.5 | valid tokens/s: 75437.5 | physical tokens/s: 76129.2 | loss tokens/s: 58528.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.766322E-01 | lm-global-supervised-tokens: 1.301661E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.358406E-02 | lm-token-buffer-mib: 2.345388E+02 | global_load_balancing_loss: 1.001106E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11174.900 | params rms: 0.060508 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:21:37.419995 | finish at 2026-10-17 18:36:41 + [2026-10-15 01:18:45] iteration 1327/ 2384 | consumed samples: 42464 | elapsed time per iteration (ms): 220564.2 | throughput per GPU (TFLOP/s/GPU): 17.2 | MFU 5.52% | FLOPs estimator: workload-aware | logical tokens/s: 76065.0 | valid tokens/s: 76065.0 | physical tokens/s: 76843.5 | loss tokens/s: 57562.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.433792E-01 | lm-global-supervised-tokens: 1.269631E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.570272E-02 | lm-token-buffer-mib: 2.340565E+02 | global_load_balancing_loss: 1.002286E+00 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11174.869 | params rms: 0.060508 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:45:36.349744 | finish at 2026-10-17 18:04:21 + [2026-10-15 01:22:31] iteration 1328/ 2384 | consumed samples: 42496 | elapsed time per iteration (ms): 226029.1 | throughput per GPU (TFLOP/s/GPU): 25.1 | MFU 8.03% | FLOPs estimator: workload-aware | logical tokens/s: 74225.5 | valid tokens/s: 74225.5 | physical tokens/s: 75048.5 | loss tokens/s: 52940.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.323702E-01 | lm-global-supervised-tokens: 1.196602E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.549412E-02 | lm-token-buffer-mib: 2.351256E+02 | global_load_balancing_loss: 1.003040E+00 | loss scale: 1.0 | grad norm: 0.176 | params norm: 11174.834 | params rms: 0.060508 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 18:18:06.759247 | finish at 2026-10-17 19:40:37 + [2026-10-15 01:26:17] iteration 1329/ 2384 | consumed samples: 42528 | elapsed time per iteration (ms): 226135.7 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.12% | FLOPs estimator: workload-aware | logical tokens/s: 74190.7 | valid tokens/s: 74190.7 | physical tokens/s: 74631.9 | loss tokens/s: 50127.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.695965E-01 | lm-global-supervised-tokens: 1.133549E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.867282E-02 | lm-token-buffer-mib: 2.334286E+02 | global_load_balancing_loss: 1.002077E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11174.792 | params rms: 0.060508 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 18:16:13.150659 | finish at 2026-10-17 19:42:30 + [2026-10-15 01:30:23] iteration 1330/ 2384 | consumed samples: 42560 | elapsed time per iteration (ms): 246360.2 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.05% | FLOPs estimator: workload-aware | logical tokens/s: 68100.3 | valid tokens/s: 68100.3 | physical tokens/s: 68632.4 | loss tokens/s: 50638.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.040542E-01 | lm-global-supervised-tokens: 1.247525E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.473010E-02 | lm-token-buffer-mib: 2.354156E+02 | global_load_balancing_loss: 9.998072E-01 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11174.752 | params rms: 0.060508 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 0:07:43.600466 | finish at 2026-10-18 01:38:07 + [2026-10-15 01:34:04] iteration 1331/ 2384 | consumed samples: 42592 | elapsed time per iteration (ms): 220501.8 | throughput per GPU (TFLOP/s/GPU): 23.4 | MFU 7.52% | FLOPs estimator: workload-aware | logical tokens/s: 76086.4 | valid tokens/s: 76086.4 | physical tokens/s: 76472.2 | loss tokens/s: 57979.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.739375E-01 | lm-global-supervised-tokens: 1.278465E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.658291E-02 | lm-token-buffer-mib: 2.330018E+02 | global_load_balancing_loss: 9.984961E-01 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11174.715 | params rms: 0.060507 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:29:48.447683 | finish at 2026-10-17 18:03:52 + [2026-10-15 01:37:47] iteration 1332/ 2384 | consumed samples: 42624 | elapsed time per iteration (ms): 223884.3 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.92% | FLOPs estimator: workload-aware | logical tokens/s: 74937.0 | valid tokens/s: 74937.0 | physical tokens/s: 75422.1 | loss tokens/s: 58107.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.209375E-01 | lm-global-supervised-tokens: 1.300939E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.578991E-02 | lm-token-buffer-mib: 2.336005E+02 | global_load_balancing_loss: 9.982142E-01 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11174.678 | params rms: 0.060507 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:25:26.268043 | finish at 2026-10-17 19:03:14 + [2026-10-15 01:41:29] iteration 1333/ 2384 | consumed samples: 42656 | elapsed time per iteration (ms): 221426.0 | throughput per GPU (TFLOP/s/GPU): 24.9 | MFU 7.97% | FLOPs estimator: workload-aware | logical tokens/s: 75768.6 | valid tokens/s: 75768.6 | physical tokens/s: 76142.7 | loss tokens/s: 57781.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.409469E-01 | lm-global-supervised-tokens: 1.279428E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.087177E-02 | lm-token-buffer-mib: 2.323432E+02 | global_load_balancing_loss: 9.984749E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11174.645 | params rms: 0.060507 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:38:38.707832 | finish at 2026-10-17 18:20:08 + [2026-10-15 01:45:18] iteration 1334/ 2384 | consumed samples: 42688 | elapsed time per iteration (ms): 228853.6 | throughput per GPU (TFLOP/s/GPU): 24.9 | MFU 7.99% | FLOPs estimator: workload-aware | logical tokens/s: 73309.6 | valid tokens/s: 73309.5 | physical tokens/s: 73901.8 | loss tokens/s: 55900.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.748140E-01 | lm-global-supervised-tokens: 1.279292E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.791022E-02 | lm-token-buffer-mib: 2.340150E+02 | global_load_balancing_loss: 9.998721E-01 | loss scale: 1.0 | grad norm: 0.135 | params norm: 11174.614 | params rms: 0.060507 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 18:44:56.326089 | finish at 2026-10-17 20:30:14 + [2026-10-15 01:49:02] iteration 1335/ 2384 | consumed samples: 42720 | elapsed time per iteration (ms): 224321.7 | throughput per GPU (TFLOP/s/GPU): 19.6 | MFU 6.28% | FLOPs estimator: workload-aware | logical tokens/s: 74790.9 | valid tokens/s: 74790.9 | physical tokens/s: 75421.7 | loss tokens/s: 54595.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.535495E-01 | lm-global-supervised-tokens: 1.224701E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.276713E-02 | lm-token-buffer-mib: 2.344555E+02 | global_load_balancing_loss: 1.001301E+00 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11174.582 | params rms: 0.060507 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:21:53.419133 | finish at 2026-10-17 19:10:55 + [2026-10-15 01:52:45] iteration 1336/ 2384 | consumed samples: 42752 | elapsed time per iteration (ms): 222591.9 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.42% | FLOPs estimator: workload-aware | logical tokens/s: 75371.8 | valid tokens/s: 75371.8 | physical tokens/s: 76009.2 | loss tokens/s: 55764.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.876582E-01 | lm-global-supervised-tokens: 1.241262E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.325289E-02 | lm-token-buffer-mib: 2.340708E+02 | global_load_balancing_loss: 1.001109E+00 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11174.546 | params rms: 0.060507 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:47:56.309816 | finish at 2026-10-17 18:40:41 + [2026-10-15 01:56:25] iteration 1337/ 2384 | consumed samples: 42784 | elapsed time per iteration (ms): 219984.8 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.72% | FLOPs estimator: workload-aware | logical tokens/s: 76265.3 | valid tokens/s: 76265.3 | physical tokens/s: 76639.8 | loss tokens/s: 55676.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.626391E-01 | lm-global-supervised-tokens: 1.224797E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.535417E-02 | lm-token-buffer-mib: 2.325831E+02 | global_load_balancing_loss: 1.001782E+00 | loss scale: 1.0 | grad norm: 0.106 | params norm: 11174.509 | params rms: 0.060506 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:58:44.099185 | finish at 2026-10-17 17:55:09 + [2026-10-15 02:00:06] iteration 1338/ 2384 | consumed samples: 42816 | elapsed time per iteration (ms): 221627.2 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.36% | FLOPs estimator: workload-aware | logical tokens/s: 75700.1 | valid tokens/s: 75700.1 | physical tokens/s: 76245.1 | loss tokens/s: 56565.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.059709E-01 | lm-global-supervised-tokens: 1.253647E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.210599E-02 | lm-token-buffer-mib: 2.333287E+02 | global_load_balancing_loss: 1.001648E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11174.474 | params rms: 0.060506 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:23:42.023401 | finish at 2026-10-17 18:23:48 + [2026-10-15 02:03:48] iteration 1339/ 2384 | consumed samples: 42848 | elapsed time per iteration (ms): 221840.3 | throughput per GPU (TFLOP/s/GPU): 24.5 | MFU 7.84% | FLOPs estimator: workload-aware | logical tokens/s: 75627.2 | valid tokens/s: 75627.2 | physical tokens/s: 76193.8 | loss tokens/s: 56384.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.065047E-01 | lm-global-supervised-tokens: 1.250840E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.760051E-02 | lm-token-buffer-mib: 2.330126E+02 | global_load_balancing_loss: 1.000039E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11174.442 | params rms: 0.060506 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:23:43.138502 | finish at 2026-10-17 18:27:31 + [2026-10-15 02:07:53] iteration 1340/ 2384 | consumed samples: 42880 | elapsed time per iteration (ms): 244690.6 | throughput per GPU (TFLOP/s/GPU): 19.5 | MFU 6.24% | FLOPs estimator: workload-aware | logical tokens/s: 68565.0 | valid tokens/s: 68565.0 | physical tokens/s: 69388.9 | loss tokens/s: 48301.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.417865E-01 | lm-global-supervised-tokens: 1.181881E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.700114E-02 | lm-token-buffer-mib: 2.331118E+02 | global_load_balancing_loss: 1.000820E+00 | loss scale: 1.0 | grad norm: 0.128 | params norm: 11174.406 | params rms: 0.060506 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:57:36.984572 | finish at 2026-10-18 01:05:30 + [2026-10-15 02:11:34] iteration 1341/ 2384 | consumed samples: 42912 | elapsed time per iteration (ms): 220794.1 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.12% | FLOPs estimator: workload-aware | logical tokens/s: 75985.7 | valid tokens/s: 75985.7 | physical tokens/s: 76166.4 | loss tokens/s: 55130.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.884736E-01 | lm-global-supervised-tokens: 1.217244E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.330115E-02 | lm-token-buffer-mib: 2.323324E+02 | global_load_balancing_loss: 9.996145E-01 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11174.369 | params rms: 0.060506 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:58:08.240629 | finish at 2026-10-17 18:09:42 + [2026-10-15 02:15:17] iteration 1342/ 2384 | consumed samples: 42944 | elapsed time per iteration (ms): 223785.5 | throughput per GPU (TFLOP/s/GPU): 25.8 | MFU 8.26% | FLOPs estimator: workload-aware | logical tokens/s: 74969.8 | valid tokens/s: 74969.8 | physical tokens/s: 75349.0 | loss tokens/s: 51554.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.186714E-01 | lm-global-supervised-tokens: 1.153712E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.447442E-02 | lm-token-buffer-mib: 2.327705E+02 | global_load_balancing_loss: 1.000246E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11174.333 | params rms: 0.060505 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:46:24.485338 | finish at 2026-10-17 19:01:42 + [2026-10-15 02:19:02] iteration 1343/ 2384 | consumed samples: 42976 | elapsed time per iteration (ms): 224326.6 | throughput per GPU (TFLOP/s/GPU): 19.7 | MFU 6.32% | FLOPs estimator: workload-aware | logical tokens/s: 74789.1 | valid tokens/s: 74789.1 | physical tokens/s: 76493.9 | loss tokens/s: 58016.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.717284E-01 | lm-global-supervised-tokens: 1.301468E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.086033E-02 | lm-token-buffer-mib: 2.374071E+02 | global_load_balancing_loss: 9.998394E-01 | loss scale: 1.0 | grad norm: 0.118 | params norm: 11174.299 | params rms: 0.060505 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:52:03.987700 | finish at 2026-10-17 19:11:06 + [2026-10-15 02:22:43] iteration 1344/ 2384 | consumed samples: 43008 | elapsed time per iteration (ms): 221649.9 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.64% | FLOPs estimator: workload-aware | logical tokens/s: 75692.2 | valid tokens/s: 75692.2 | physical tokens/s: 76446.6 | loss tokens/s: 59104.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.292914E-01 | lm-global-supervised-tokens: 1.310058E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.674093E-02 | lm-token-buffer-mib: 2.351181E+02 | global_load_balancing_loss: 9.992916E-01 | loss scale: 1.0 | grad norm: 0.276 | params norm: 11174.265 | params rms: 0.060505 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:01:55.911827 | finish at 2026-10-17 18:24:39 + [2026-10-15 02:26:31] iteration 1345/ 2384 | consumed samples: 43040 | elapsed time per iteration (ms): 227329.4 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.88% | FLOPs estimator: workload-aware | logical tokens/s: 73801.3 | valid tokens/s: 73801.3 | physical tokens/s: 74401.8 | loss tokens/s: 54795.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.803680E-01 | lm-global-supervised-tokens: 1.245659E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.769125E-02 | lm-token-buffer-mib: 2.344278E+02 | global_load_balancing_loss: 1.001299E+00 | loss scale: 1.0 | grad norm: 0.137 | params norm: 11174.231 | params rms: 0.060505 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:36:35.275152 | finish at 2026-10-17 20:03:06 + [2026-10-15 02:30:12] iteration 1346/ 2384 | consumed samples: 43072 | elapsed time per iteration (ms): 220968.3 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.52% | FLOPs estimator: workload-aware | logical tokens/s: 75925.8 | valid tokens/s: 75925.8 | physical tokens/s: 76580.8 | loss tokens/s: 59823.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.692812E-01 | lm-global-supervised-tokens: 1.321912E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.370442E-02 | lm-token-buffer-mib: 2.330796E+02 | global_load_balancing_loss: 1.000819E+00 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11174.200 | params rms: 0.060505 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:42:45.117286 | finish at 2026-10-17 18:12:57 + [2026-10-15 02:33:56] iteration 1347/ 2384 | consumed samples: 43104 | elapsed time per iteration (ms): 224402.6 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 7.00% | FLOPs estimator: workload-aware | logical tokens/s: 74763.8 | valid tokens/s: 74763.8 | physical tokens/s: 75254.2 | loss tokens/s: 55860.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.009840E-01 | lm-global-supervised-tokens: 1.253525E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.715902E-02 | lm-token-buffer-mib: 2.344370E+02 | global_load_balancing_loss: 1.001138E+00 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11174.167 | params rms: 0.060504 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:38:25.491060 | finish at 2026-10-17 19:12:22 + [2026-10-15 02:37:38] iteration 1348/ 2384 | consumed samples: 43136 | elapsed time per iteration (ms): 221558.9 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.34% | FLOPs estimator: workload-aware | logical tokens/s: 75723.4 | valid tokens/s: 75723.4 | physical tokens/s: 75955.7 | loss tokens/s: 57144.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.793924E-01 | lm-global-supervised-tokens: 1.266092E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.662691E-02 | lm-token-buffer-mib: 2.328326E+02 | global_load_balancing_loss: 9.996552E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11174.133 | params rms: 0.060504 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:45:35.072145 | finish at 2026-10-17 18:23:13 + [2026-10-15 02:41:20] iteration 1349/ 2384 | consumed samples: 43168 | elapsed time per iteration (ms): 222467.0 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.69% | FLOPs estimator: workload-aware | logical tokens/s: 75414.4 | valid tokens/s: 75414.3 | physical tokens/s: 76070.0 | loss tokens/s: 58511.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.917353E-01 | lm-global-supervised-tokens: 1.301684E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.445179E-02 | lm-token-buffer-mib: 2.338013E+02 | global_load_balancing_loss: 1.001964E+00 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11174.099 | params rms: 0.060504 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:57:33.310461 | finish at 2026-10-17 18:38:53 + [2026-10-15 02:45:26] iteration 1350/ 2384 | consumed samples: 43200 | elapsed time per iteration (ms): 245449.6 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.41% | FLOPs estimator: workload-aware | logical tokens/s: 68352.9 | valid tokens/s: 68352.9 | physical tokens/s: 69103.9 | loss tokens/s: 56576.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.965923E-01 | lm-global-supervised-tokens: 1.388665E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.675910E-02 | lm-token-buffer-mib: 2.356849E+02 | global_load_balancing_loss: 9.995879E-01 | loss scale: 1.0 | grad norm: 0.108 | params norm: 11174.065 | params rms: 0.060504 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:29:54.924099 | finish at 2026-10-18 01:15:20 + [2026-10-15 02:49:08] iteration 1351/ 2384 | consumed samples: 43232 | elapsed time per iteration (ms): 222693.3 | throughput per GPU (TFLOP/s/GPU): 23.4 | MFU 7.50% | FLOPs estimator: workload-aware | logical tokens/s: 75337.7 | valid tokens/s: 75337.7 | physical tokens/s: 75776.1 | loss tokens/s: 53473.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.611274E-01 | lm-global-supervised-tokens: 1.190826E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.278993E-02 | lm-token-buffer-mib: 2.335067E+02 | global_load_balancing_loss: 9.997407E-01 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11174.028 | params rms: 0.060504 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:54:02.138764 | finish at 2026-10-17 18:43:10 + [2026-10-15 02:52:51] iteration 1352/ 2384 | consumed samples: 43264 | elapsed time per iteration (ms): 222508.9 | throughput per GPU (TFLOP/s/GPU): 24.7 | MFU 7.92% | FLOPs estimator: workload-aware | logical tokens/s: 75400.1 | valid tokens/s: 75400.1 | physical tokens/s: 76247.8 | loss tokens/s: 59393.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.336757E-01 | lm-global-supervised-tokens: 1.321552E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.370580E-02 | lm-token-buffer-mib: 2.327349E+02 | global_load_balancing_loss: 9.999384E-01 | loss scale: 1.0 | grad norm: 0.253 | params norm: 11173.991 | params rms: 0.060504 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:47:09.209084 | finish at 2026-10-17 18:40:00 + [2026-10-15 02:56:31] iteration 1353/ 2384 | consumed samples: 43296 | elapsed time per iteration (ms): 219914.4 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.16% | FLOPs estimator: workload-aware | logical tokens/s: 76289.5 | valid tokens/s: 76289.5 | physical tokens/s: 77274.6 | loss tokens/s: 60536.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.160543E-01 | lm-global-supervised-tokens: 1.331278E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.767663E-02 | lm-token-buffer-mib: 2.337880E+02 | global_load_balancing_loss: 9.994514E-01 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11173.948 | params rms: 0.060503 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:58:51.723398 | finish at 2026-10-17 17:55:22 + [2026-10-15 03:00:13] iteration 1354/ 2384 | consumed samples: 43328 | elapsed time per iteration (ms): 222437.7 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.73% | FLOPs estimator: workload-aware | logical tokens/s: 75424.3 | valid tokens/s: 75424.3 | physical tokens/s: 76171.5 | loss tokens/s: 53623.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.538190E-01 | lm-global-supervised-tokens: 1.192779E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.937989E-02 | lm-token-buffer-mib: 2.342800E+02 | global_load_balancing_loss: 9.999251E-01 | loss scale: 1.0 | grad norm: 0.488 | params norm: 11173.907 | params rms: 0.060503 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:38:30.871308 | finish at 2026-10-17 18:38:44 + [2026-10-15 03:03:57] iteration 1355/ 2384 | consumed samples: 43360 | elapsed time per iteration (ms): 223821.9 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.97% | FLOPs estimator: workload-aware | logical tokens/s: 74957.8 | valid tokens/s: 74957.8 | physical tokens/s: 75367.9 | loss tokens/s: 53320.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.154784E-01 | lm-global-supervised-tokens: 1.193430E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.006813E-02 | lm-token-buffer-mib: 2.335209E+02 | global_load_balancing_loss: 9.994422E-01 | loss scale: 1.0 | grad norm: 0.143 | params norm: 11173.869 | params rms: 0.060503 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:58:32.741857 | finish at 2026-10-17 19:02:30 + [2026-10-15 03:07:40] iteration 1356/ 2384 | consumed samples: 43392 | elapsed time per iteration (ms): 222983.1 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.69% | FLOPs estimator: workload-aware | logical tokens/s: 75239.7 | valid tokens/s: 75239.7 | physical tokens/s: 76114.1 | loss tokens/s: 57620.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.402177E-01 | lm-global-supervised-tokens: 1.284831E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.472212E-02 | lm-token-buffer-mib: 2.359106E+02 | global_load_balancing_loss: 1.001122E+00 | loss scale: 1.0 | grad norm: 0.669 | params norm: 11173.832 | params rms: 0.060503 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:40:26.646098 | finish at 2026-10-17 18:48:07 + [2026-10-15 03:11:23] iteration 1357/ 2384 | consumed samples: 43424 | elapsed time per iteration (ms): 223213.8 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.54% | FLOPs estimator: workload-aware | logical tokens/s: 75162.0 | valid tokens/s: 75162.0 | physical tokens/s: 75950.0 | loss tokens/s: 59640.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.852907E-01 | lm-global-supervised-tokens: 1.331264E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.404932E-02 | lm-token-buffer-mib: 2.334046E+02 | global_load_balancing_loss: 1.002902E+00 | loss scale: 1.0 | grad norm: 0.288 | params norm: 11173.794 | params rms: 0.060502 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:40:40.529458 | finish at 2026-10-17 18:52:04 + [2026-10-15 03:15:04] iteration 1358/ 2384 | consumed samples: 43456 | elapsed time per iteration (ms): 220989.2 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.84% | FLOPs estimator: workload-aware | logical tokens/s: 75918.6 | valid tokens/s: 75918.6 | physical tokens/s: 76655.8 | loss tokens/s: 60702.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.241675E-01 | lm-global-supervised-tokens: 1.341452E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.284406E-02 | lm-token-buffer-mib: 2.350590E+02 | global_load_balancing_loss: 1.000199E+00 | loss scale: 1.0 | grad norm: 0.547 | params norm: 11173.753 | params rms: 0.060502 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:58:54.944269 | finish at 2026-10-17 18:13:59 + [2026-10-15 03:18:50] iteration 1359/ 2384 | consumed samples: 43488 | elapsed time per iteration (ms): 225987.1 | throughput per GPU (TFLOP/s/GPU): 24.6 | MFU 7.89% | FLOPs estimator: workload-aware | logical tokens/s: 74239.5 | valid tokens/s: 74239.5 | physical tokens/s: 75007.2 | loss tokens/s: 55109.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.449207E-01 | lm-global-supervised-tokens: 1.245404E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.390692E-02 | lm-token-buffer-mib: 2.332833E+02 | global_load_balancing_loss: 1.000884E+00 | loss scale: 1.0 | grad norm: 0.145 | params norm: 11173.713 | params rms: 0.060502 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:20:36.792046 | finish at 2026-10-17 19:39:27 + [2026-10-15 03:22:53] iteration 1360/ 2384 | consumed samples: 43520 | elapsed time per iteration (ms): 243015.0 | throughput per GPU (TFLOP/s/GPU): 19.2 | MFU 6.16% | FLOPs estimator: workload-aware | logical tokens/s: 69037.6 | valid tokens/s: 69037.6 | physical tokens/s: 69298.5 | loss tokens/s: 52183.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.136057E-01 | lm-global-supervised-tokens: 1.268145E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.944521E-02 | lm-token-buffer-mib: 2.330587E+02 | global_load_balancing_loss: 1.000546E+00 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11173.675 | params rms: 0.060502 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:07:27.312500 | finish at 2026-10-18 00:30:20 +times across ranks (ms): + save-checkpoint: + rank 0: 24894.26 + rank 1: 24894.28 + rank 2: 24894.27 + rank 3: 24894.27 + rank 4: 24894.26 + rank 5: 24894.32 + rank 6: 24894.31 + rank 7: 24894.27 + rank 8: 24894.39 + rank 9: 24894.40 + rank 10: 24894.40 + rank 11: 24894.40 + rank 12: 24894.42 + rank 13: 24894.38 + rank 14: 24894.38 + rank 15: 24894.39 + rank 16: 24896.37 + rank 17: 24896.34 + rank 18: 24896.37 + rank 19: 24896.34 + rank 20: 24896.42 + rank 21: 24896.34 + rank 22: 24896.36 + rank 23: 24896.42 + rank 24: 24894.41 + rank 25: 24894.41 + rank 26: 24894.38 + rank 27: 24894.37 + rank 28: 24894.42 + rank 29: 24894.44 + rank 30: 24894.41 + rank 31: 24894.44 + rank 32: 24894.01 + rank 33: 24894.04 + rank 34: 24894.00 + rank 35: 24894.00 + rank 36: 24893.98 + rank 37: 24894.03 + rank 38: 24893.97 + rank 39: 24893.99 + rank 40: 24894.03 + rank 41: 24894.03 + rank 42: 24894.00 + rank 43: 24893.97 + rank 44: 24893.98 + rank 45: 24893.79 + rank 46: 24893.98 + rank 47: 24893.98 + rank 48: 24894.71 + rank 49: 24894.76 + rank 50: 24894.75 + rank 51: 24894.71 + rank 52: 24894.73 + rank 53: 24894.73 + rank 54: 24894.71 + rank 55: 24894.71 + rank 56: 24894.79 + rank 57: 24894.77 + rank 58: 24894.77 + rank 59: 24894.82 + rank 60: 24894.75 + rank 61: 24894.77 + rank 62: 24894.77 + rank 63: 24894.76 + [2026-10-15 03:27:19] iteration 1361/ 2384 | consumed samples: 43552 | elapsed time per iteration (ms): 241079.2 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.60% | FLOPs estimator: workload-aware | logical tokens/s: 69592.0 | valid tokens/s: 69592.0 | physical tokens/s: 69942.8 | loss tokens/s: 52437.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.215577E-01 | lm-global-supervised-tokens: 1.264162E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 2.188102E-01 | lm-token-buffer-mib: 2.328470E+02 | global_load_balancing_loss: 9.998307E-01 | loss scale: 1.0 | grad norm: 0.299 | params norm: 11173.637 | params rms: 0.060502 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:30:24.010167 | finish at 2026-10-17 23:57:43 + [2026-10-15 03:31:03] iteration 1362/ 2384 | consumed samples: 43584 | elapsed time per iteration (ms): 223895.8 | throughput per GPU (TFLOP/s/GPU): 18.6 | MFU 5.97% | FLOPs estimator: workload-aware | logical tokens/s: 74933.0 | valid tokens/s: 74933.0 | physical tokens/s: 75512.8 | loss tokens/s: 56436.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.635044E-01 | lm-global-supervised-tokens: 1.263579E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.940023E-02 | lm-token-buffer-mib: 2.330571E+02 | global_load_balancing_loss: 9.991987E-01 | loss scale: 1.0 | grad norm: 0.213 | params norm: 11173.601 | params rms: 0.060501 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:33:41.518925 | finish at 2026-10-17 19:04:45 + [2026-10-15 03:34:49] iteration 1363/ 2384 | consumed samples: 43616 | elapsed time per iteration (ms): 225717.3 | throughput per GPU (TFLOP/s/GPU): 27.2 | MFU 8.72% | FLOPs estimator: workload-aware | logical tokens/s: 74328.2 | valid tokens/s: 74328.2 | physical tokens/s: 75083.2 | loss tokens/s: 55512.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.648263E-01 | lm-global-supervised-tokens: 1.253005E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.908307E-02 | lm-token-buffer-mib: 2.363022E+02 | global_load_balancing_loss: 9.995334E-01 | loss scale: 1.0 | grad norm: 0.153 | params norm: 11173.561 | params rms: 0.060501 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:00:57.399994 | finish at 2026-10-17 19:35:46 + [2026-10-15 03:38:29] iteration 1364/ 2384 | consumed samples: 43648 | elapsed time per iteration (ms): 220248.6 | throughput per GPU (TFLOP/s/GPU): 27.0 | MFU 8.66% | FLOPs estimator: workload-aware | logical tokens/s: 76173.7 | valid tokens/s: 76173.7 | physical tokens/s: 76544.2 | loss tokens/s: 56415.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.897565E-01 | lm-global-supervised-tokens: 1.242541E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.359088E-02 | lm-token-buffer-mib: 2.326858E+02 | global_load_balancing_loss: 9.998620E-01 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11173.519 | params rms: 0.060501 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:24:13.531880 | finish at 2026-10-17 18:02:43 + [2026-10-15 03:42:12] iteration 1365/ 2384 | consumed samples: 43680 | elapsed time per iteration (ms): 222837.9 | throughput per GPU (TFLOP/s/GPU): 24.0 | MFU 7.70% | FLOPs estimator: workload-aware | logical tokens/s: 75288.7 | valid tokens/s: 75288.7 | physical tokens/s: 75981.0 | loss tokens/s: 60371.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.720353E-01 | lm-global-supervised-tokens: 1.345306E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.520604E-02 | lm-token-buffer-mib: 2.353610E+02 | global_load_balancing_loss: 1.001240E+00 | loss scale: 1.0 | grad norm: 0.172 | params norm: 11173.478 | params rms: 0.060501 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:04:31.837757 | finish at 2026-10-17 18:46:44 + [2026-10-15 03:45:57] iteration 1366/ 2384 | consumed samples: 43712 | elapsed time per iteration (ms): 224854.7 | throughput per GPU (TFLOP/s/GPU): 24.5 | MFU 7.87% | FLOPs estimator: workload-aware | logical tokens/s: 74613.5 | valid tokens/s: 74613.5 | physical tokens/s: 75229.3 | loss tokens/s: 58424.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.484716E-01 | lm-global-supervised-tokens: 1.313690E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.766376E-02 | lm-token-buffer-mib: 2.354095E+02 | global_load_balancing_loss: 1.000129E+00 | loss scale: 1.0 | grad norm: 0.138 | params norm: 11173.437 | params rms: 0.060501 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:35:02.050953 | finish at 2026-10-17 19:20:59 + [2026-10-15 03:49:39] iteration 1367/ 2384 | consumed samples: 43744 | elapsed time per iteration (ms): 222164.5 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.83% | FLOPs estimator: workload-aware | logical tokens/s: 75516.9 | valid tokens/s: 75516.9 | physical tokens/s: 76730.7 | loss tokens/s: 61450.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.977773E-01 | lm-global-supervised-tokens: 1.365218E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.375826E-02 | lm-token-buffer-mib: 2.350686E+02 | global_load_balancing_loss: 1.002572E+00 | loss scale: 1.0 | grad norm: 0.284 | params norm: 11173.394 | params rms: 0.060500 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:45:41.316381 | finish at 2026-10-17 18:35:20 + [2026-10-15 03:53:22] iteration 1368/ 2384 | consumed samples: 43776 | elapsed time per iteration (ms): 223229.6 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.53% | FLOPs estimator: workload-aware | logical tokens/s: 75156.6 | valid tokens/s: 75156.6 | physical tokens/s: 75836.0 | loss tokens/s: 55508.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.420509E-01 | lm-global-supervised-tokens: 1.239116E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.896137E-02 | lm-token-buffer-mib: 2.327231E+02 | global_load_balancing_loss: 1.001309E+00 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11173.351 | params rms: 0.060500 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:00:01.305527 | finish at 2026-10-17 18:53:23 + [2026-10-15 03:57:04] iteration 1369/ 2384 | consumed samples: 43808 | elapsed time per iteration (ms): 221759.8 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.73% | FLOPs estimator: workload-aware | logical tokens/s: 75654.7 | valid tokens/s: 75654.7 | physical tokens/s: 76497.1 | loss tokens/s: 59342.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.337918E-01 | lm-global-supervised-tokens: 1.315970E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.351155E-02 | lm-token-buffer-mib: 2.360365E+02 | global_load_balancing_loss: 1.000581E+00 | loss scale: 1.0 | grad norm: 0.122 | params norm: 11173.310 | params rms: 0.060500 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:31:26.159689 | finish at 2026-10-17 18:28:30 + [2026-10-15 04:01:07] iteration 1370/ 2384 | consumed samples: 43840 | elapsed time per iteration (ms): 243372.9 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.58% | FLOPs estimator: workload-aware | logical tokens/s: 68936.1 | valid tokens/s: 68936.1 | physical tokens/s: 69411.7 | loss tokens/s: 50462.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.468635E-01 | lm-global-supervised-tokens: 1.228120E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.294469E-02 | lm-token-buffer-mib: 2.338495E+02 | global_load_balancing_loss: 1.000412E+00 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11173.272 | params rms: 0.060500 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:33:00.077335 | finish at 2026-10-18 00:34:07 + [2026-10-15 04:04:49] iteration 1371/ 2384 | consumed samples: 43872 | elapsed time per iteration (ms): 222225.0 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.39% | FLOPs estimator: workload-aware | logical tokens/s: 75496.5 | valid tokens/s: 75496.5 | physical tokens/s: 76247.9 | loss tokens/s: 57619.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.276688E-01 | lm-global-supervised-tokens: 1.280442E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.908706E-02 | lm-token-buffer-mib: 2.343177E+02 | global_load_balancing_loss: 1.000531E+00 | loss scale: 1.0 | grad norm: 0.115 | params norm: 11173.238 | params rms: 0.060499 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:31:53.940844 | finish at 2026-10-17 18:36:43 + [2026-10-15 04:08:32] iteration 1372/ 2384 | consumed samples: 43904 | elapsed time per iteration (ms): 222414.5 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.55% | FLOPs estimator: workload-aware | logical tokens/s: 75432.0 | valid tokens/s: 75432.0 | physical tokens/s: 76309.3 | loss tokens/s: 56621.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.218849E-01 | lm-global-supervised-tokens: 1.259349E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.286239E-02 | lm-token-buffer-mib: 2.358045E+02 | global_load_balancing_loss: 9.994797E-01 | loss scale: 1.0 | grad norm: 0.096 | params norm: 11173.206 | params rms: 0.060499 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:31:23.519359 | finish at 2026-10-17 18:39:55 + [2026-10-15 04:12:19] iteration 1373/ 2384 | consumed samples: 43936 | elapsed time per iteration (ms): 226955.4 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.53% | FLOPs estimator: workload-aware | logical tokens/s: 73923.0 | valid tokens/s: 73922.9 | physical tokens/s: 75279.9 | loss tokens/s: 59756.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.146881E-01 | lm-global-supervised-tokens: 1.356201E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.407537E-02 | lm-token-buffer-mib: 2.370708E+02 | global_load_balancing_loss: 1.000170E+00 | loss scale: 1.0 | grad norm: 0.103 | params norm: 11173.178 | params rms: 0.060499 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:44:11.924576 | finish at 2026-10-17 19:56:31 + [2026-10-15 04:16:03] iteration 1374/ 2384 | consumed samples: 43968 | elapsed time per iteration (ms): 223800.1 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.64% | FLOPs estimator: workload-aware | logical tokens/s: 74965.2 | valid tokens/s: 74965.2 | physical tokens/s: 75322.9 | loss tokens/s: 56380.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.531268E-01 | lm-global-supervised-tokens: 1.261801E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.066247E-02 | lm-token-buffer-mib: 2.324407E+02 | global_load_balancing_loss: 1.000933E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11173.151 | params rms: 0.060499 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:47:18.066654 | finish at 2026-10-17 19:03:21 + [2026-10-15 04:19:48] iteration 1375/ 2384 | consumed samples: 44000 | elapsed time per iteration (ms): 225604.8 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.46% | FLOPs estimator: workload-aware | logical tokens/s: 74365.4 | valid tokens/s: 74365.4 | physical tokens/s: 75453.1 | loss tokens/s: 55884.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.542847E-01 | lm-global-supervised-tokens: 1.260791E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.619287E-02 | lm-token-buffer-mib: 2.333170E+02 | global_load_balancing_loss: 1.001319E+00 | loss scale: 1.0 | grad norm: 0.183 | params norm: 11173.121 | params rms: 0.060499 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:13:55.260988 | finish at 2026-10-17 19:33:43 + [2026-10-15 04:23:31] iteration 1376/ 2384 | consumed samples: 44032 | elapsed time per iteration (ms): 222535.9 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.34% | FLOPs estimator: workload-aware | logical tokens/s: 75391.0 | valid tokens/s: 75391.0 | physical tokens/s: 75961.6 | loss tokens/s: 52481.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.669555E-01 | lm-global-supervised-tokens: 1.167892E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.106304E-02 | lm-token-buffer-mib: 2.345094E+02 | global_load_balancing_loss: 9.996012E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11173.089 | params rms: 0.060499 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:18:36.206543 | finish at 2026-10-17 18:42:07 + [2026-10-15 04:27:11] iteration 1377/ 2384 | consumed samples: 44064 | elapsed time per iteration (ms): 220526.7 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.50% | FLOPs estimator: workload-aware | logical tokens/s: 76077.9 | valid tokens/s: 76077.9 | physical tokens/s: 76578.0 | loss tokens/s: 53447.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.814720E-01 | lm-global-supervised-tokens: 1.178657E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.130437E-02 | lm-token-buffer-mib: 2.331847E+02 | global_load_balancing_loss: 1.000107E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11173.054 | params rms: 0.060498 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:41:10.405647 | finish at 2026-10-17 18:08:22 + [2026-10-15 04:30:54] iteration 1378/ 2384 | consumed samples: 44096 | elapsed time per iteration (ms): 222460.9 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.44% | FLOPs estimator: workload-aware | logical tokens/s: 75416.4 | valid tokens/s: 75416.4 | physical tokens/s: 76022.1 | loss tokens/s: 54182.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.658497E-01 | lm-global-supervised-tokens: 1.205356E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.895193E-02 | lm-token-buffer-mib: 2.343060E+02 | global_load_balancing_loss: 9.996681E-01 | loss scale: 1.0 | grad norm: 0.145 | params norm: 11173.021 | params rms: 0.060498 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:09:55.647000 | finish at 2026-10-17 18:40:49 + [2026-10-15 04:34:40] iteration 1379/ 2384 | consumed samples: 44128 | elapsed time per iteration (ms): 226199.3 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.30% | FLOPs estimator: workload-aware | logical tokens/s: 74169.9 | valid tokens/s: 74169.9 | physical tokens/s: 75805.4 | loss tokens/s: 58872.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.669855E-01 | lm-global-supervised-tokens: 1.331687E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.741751E-02 | lm-token-buffer-mib: 2.391747E+02 | global_load_balancing_loss: 9.998946E-01 | loss scale: 1.0 | grad norm: 0.152 | params norm: 11172.994 | params rms: 0.060498 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:08:50.335368 | finish at 2026-10-17 19:43:30 + [2026-10-15 04:38:41] iteration 1380/ 2384 | consumed samples: 44160 | elapsed time per iteration (ms): 241150.0 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.01% | FLOPs estimator: workload-aware | logical tokens/s: 69571.6 | valid tokens/s: 69571.6 | physical tokens/s: 70372.4 | loss tokens/s: 55899.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.147809E-01 | lm-global-supervised-tokens: 1.348010E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.212333E-02 | lm-token-buffer-mib: 2.337803E+02 | global_load_balancing_loss: 1.002000E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11172.970 | params rms: 0.060498 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:15:14.564429 | finish at 2026-10-17 23:53:56 + [2026-10-15 04:42:24] iteration 1381/ 2384 | consumed samples: 44192 | elapsed time per iteration (ms): 222969.1 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.92% | FLOPs estimator: workload-aware | logical tokens/s: 75244.4 | valid tokens/s: 75244.4 | physical tokens/s: 75829.8 | loss tokens/s: 60114.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.468270E-01 | lm-global-supervised-tokens: 1.340364E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.725868E-02 | lm-token-buffer-mib: 2.338376E+02 | global_load_balancing_loss: 1.001258E+00 | loss scale: 1.0 | grad norm: 0.122 | params norm: 11172.945 | params rms: 0.060498 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:07:17.994146 | finish at 2026-10-17 18:49:42 + [2026-10-15 04:46:07] iteration 1382/ 2384 | consumed samples: 44224 | elapsed time per iteration (ms): 222504.8 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.89% | FLOPs estimator: workload-aware | logical tokens/s: 75401.5 | valid tokens/s: 75401.5 | physical tokens/s: 76655.4 | loss tokens/s: 61061.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.308783E-01 | lm-global-supervised-tokens: 1.358655E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.681557E-02 | lm-token-buffer-mib: 2.362805E+02 | global_load_balancing_loss: 1.000137E+00 | loss scale: 1.0 | grad norm: 0.314 | params norm: 11172.915 | params rms: 0.060498 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:55:49.768591 | finish at 2026-10-17 18:41:56 + [2026-10-15 04:49:52] iteration 1383/ 2384 | consumed samples: 44256 | elapsed time per iteration (ms): 225872.4 | throughput per GPU (TFLOP/s/GPU): 24.2 | MFU 7.77% | FLOPs estimator: workload-aware | logical tokens/s: 74277.2 | valid tokens/s: 74277.2 | physical tokens/s: 75027.4 | loss tokens/s: 54209.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.035193E-01 | lm-global-supervised-tokens: 1.224434E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.249740E-02 | lm-token-buffer-mib: 2.354229E+02 | global_load_balancing_loss: 1.001508E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11172.886 | params rms: 0.060498 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:48:18.275309 | finish at 2026-10-17 19:38:11 + [2026-10-15 04:53:37] iteration 1384/ 2384 | consumed samples: 44288 | elapsed time per iteration (ms): 224897.1 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.77% | FLOPs estimator: workload-aware | logical tokens/s: 74599.4 | valid tokens/s: 74599.4 | physical tokens/s: 74989.4 | loss tokens/s: 54615.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.176722E-01 | lm-global-supervised-tokens: 1.228281E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.318817E-02 | lm-token-buffer-mib: 2.335167E+02 | global_load_balancing_loss: 9.991278E-01 | loss scale: 1.0 | grad norm: 0.179 | params norm: 11172.854 | params rms: 0.060497 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:28:17.135496 | finish at 2026-10-17 19:21:54 + [2026-10-15 04:57:18] iteration 1385/ 2384 | consumed samples: 44320 | elapsed time per iteration (ms): 221144.6 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.98% | FLOPs estimator: workload-aware | logical tokens/s: 75865.3 | valid tokens/s: 75865.3 | physical tokens/s: 76572.4 | loss tokens/s: 58705.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.014125E-01 | lm-global-supervised-tokens: 1.298232E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.976335E-02 | lm-token-buffer-mib: 2.347643E+02 | global_load_balancing_loss: 9.993963E-01 | loss scale: 1.0 | grad norm: 0.326 | params norm: 11172.823 | params rms: 0.060497 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:22:03.446502 | finish at 2026-10-17 18:19:22 + [2026-10-15 05:00:59] iteration 1386/ 2384 | consumed samples: 44352 | elapsed time per iteration (ms): 220399.8 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.14% | FLOPs estimator: workload-aware | logical tokens/s: 76121.5 | valid tokens/s: 76121.5 | physical tokens/s: 76946.7 | loss tokens/s: 59050.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.627721E-01 | lm-global-supervised-tokens: 1.301477E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.698583E-02 | lm-token-buffer-mib: 2.354031E+02 | global_load_balancing_loss: 9.998385E-01 | loss scale: 1.0 | grad norm: 0.108 | params norm: 11172.790 | params rms: 0.060497 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:05:59.012359 | finish at 2026-10-17 18:06:58 + [2026-10-15 05:04:43] iteration 1387/ 2384 | consumed samples: 44384 | elapsed time per iteration (ms): 223899.1 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.80% | FLOPs estimator: workload-aware | logical tokens/s: 74931.9 | valid tokens/s: 74931.9 | physical tokens/s: 75357.6 | loss tokens/s: 54271.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.069954E-01 | lm-global-supervised-tokens: 1.215139E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.918732E-02 | lm-token-buffer-mib: 2.334402E+02 | global_load_balancing_loss: 1.000123E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11172.756 | params rms: 0.060497 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:00:27.430101 | finish at 2026-10-17 19:05:10 + [2026-10-15 05:08:30] iteration 1388/ 2384 | consumed samples: 44416 | elapsed time per iteration (ms): 227690.5 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.38% | FLOPs estimator: workload-aware | logical tokens/s: 73684.0 | valid tokens/s: 73684.0 | physical tokens/s: 74425.8 | loss tokens/s: 55357.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.456009E-01 | lm-global-supervised-tokens: 1.260446E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.283969E-02 | lm-token-buffer-mib: 2.347693E+02 | global_load_balancing_loss: 1.000248E+00 | loss scale: 1.0 | grad norm: 0.172 | params norm: 11172.720 | params rms: 0.060497 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:59:39.748944 | finish at 2026-10-17 20:08:10 + [2026-10-15 05:12:12] iteration 1389/ 2384 | consumed samples: 44448 | elapsed time per iteration (ms): 221814.2 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.63% | FLOPs estimator: workload-aware | logical tokens/s: 75636.2 | valid tokens/s: 75636.2 | physical tokens/s: 76699.3 | loss tokens/s: 58395.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.006677E-01 | lm-global-supervised-tokens: 1.295300E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.513081E-02 | lm-token-buffer-mib: 2.352485E+02 | global_load_balancing_loss: 1.001233E+00 | loss scale: 1.0 | grad norm: 0.375 | params norm: 11172.681 | params rms: 0.060496 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:18:25.100458 | finish at 2026-10-17 18:30:37 + [2026-10-15 05:16:17] iteration 1390/ 2384 | consumed samples: 44480 | elapsed time per iteration (ms): 244868.1 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.38% | FLOPs estimator: workload-aware | logical tokens/s: 68515.1 | valid tokens/s: 68515.1 | physical tokens/s: 68736.4 | loss tokens/s: 45939.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.618429E-01 | lm-global-supervised-tokens: 1.124908E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.933844E-02 | lm-token-buffer-mib: 2.327271E+02 | global_load_balancing_loss: 1.001224E+00 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11172.642 | params rms: 0.060496 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:36:38.852699 | finish at 2026-10-18 00:52:56 + [2026-10-15 05:19:57] iteration 1391/ 2384 | consumed samples: 44512 | elapsed time per iteration (ms): 220282.2 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.56% | FLOPs estimator: workload-aware | logical tokens/s: 76162.2 | valid tokens/s: 76162.2 | physical tokens/s: 76544.2 | loss tokens/s: 54110.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.694156E-01 | lm-global-supervised-tokens: 1.191947E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.232258E-02 | lm-token-buffer-mib: 2.330801E+02 | global_load_balancing_loss: 1.001411E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11172.608 | params rms: 0.060496 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:45:40.191316 | finish at 2026-10-17 18:05:38 + [2026-10-15 05:23:42] iteration 1392/ 2384 | consumed samples: 44544 | elapsed time per iteration (ms): 224602.9 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.12% | FLOPs estimator: workload-aware | logical tokens/s: 74697.0 | valid tokens/s: 74697.0 | physical tokens/s: 75613.4 | loss tokens/s: 56832.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.074314E-01 | lm-global-supervised-tokens: 1.276484E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.905955E-02 | lm-token-buffer-mib: 2.363974E+02 | global_load_balancing_loss: 1.001004E+00 | loss scale: 1.0 | grad norm: 0.261 | params norm: 11172.572 | params rms: 0.060496 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:53:26.047501 | finish at 2026-10-17 19:17:08 + [2026-10-15 05:27:24] iteration 1393/ 2384 | consumed samples: 44576 | elapsed time per iteration (ms): 222108.6 | throughput per GPU (TFLOP/s/GPU): 26.2 | MFU 8.40% | FLOPs estimator: workload-aware | logical tokens/s: 75536.0 | valid tokens/s: 75536.0 | physical tokens/s: 75858.2 | loss tokens/s: 61614.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.570716E-01 | lm-global-supervised-tokens: 1.368521E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.750469E-02 | lm-token-buffer-mib: 2.330004E+02 | global_load_balancing_loss: 9.997817E-01 | loss scale: 1.0 | grad norm: 0.166 | params norm: 11172.535 | params rms: 0.060496 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:08:29.664559 | finish at 2026-10-17 18:35:54 + [2026-10-15 05:31:11] iteration 1394/ 2384 | consumed samples: 44608 | elapsed time per iteration (ms): 226469.0 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.36% | FLOPs estimator: workload-aware | logical tokens/s: 74081.7 | valid tokens/s: 74081.7 | physical tokens/s: 75358.8 | loss tokens/s: 53481.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.549878E-01 | lm-global-supervised-tokens: 1.211183E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.433506E-02 | lm-token-buffer-mib: 2.394083E+02 | global_load_balancing_loss: 9.997303E-01 | loss scale: 1.0 | grad norm: 0.189 | params norm: 11172.495 | params rms: 0.060495 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:16:44.287913 | finish at 2026-10-17 19:47:55 + [2026-10-15 05:34:52] iteration 1395/ 2384 | consumed samples: 44640 | elapsed time per iteration (ms): 221394.2 | throughput per GPU (TFLOP/s/GPU): 18.9 | MFU 6.06% | FLOPs estimator: workload-aware | logical tokens/s: 75779.6 | valid tokens/s: 75779.6 | physical tokens/s: 77133.6 | loss tokens/s: 59788.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.063993E-01 | lm-global-supervised-tokens: 1.323678E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.528396E-02 | lm-token-buffer-mib: 2.357366E+02 | global_load_balancing_loss: 9.996108E-01 | loss scale: 1.0 | grad norm: 0.495 | params norm: 11172.458 | params rms: 0.060495 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:49:18.902320 | finish at 2026-10-17 18:24:11 + [2026-10-15 05:38:32] iteration 1396/ 2384 | consumed samples: 44672 | elapsed time per iteration (ms): 219965.3 | throughput per GPU (TFLOP/s/GPU): 19.1 | MFU 6.13% | FLOPs estimator: workload-aware | logical tokens/s: 76272.0 | valid tokens/s: 76271.9 | physical tokens/s: 76761.5 | loss tokens/s: 56558.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.866529E-01 | lm-global-supervised-tokens: 1.244092E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.683420E-02 | lm-token-buffer-mib: 2.332464E+02 | global_load_balancing_loss: 9.995511E-01 | loss scale: 1.0 | grad norm: 0.329 | params norm: 11172.425 | params rms: 0.060495 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:22:05.730616 | finish at 2026-10-17 18:00:38 + [2026-10-15 05:42:14] iteration 1397/ 2384 | consumed samples: 44704 | elapsed time per iteration (ms): 221988.8 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.41% | FLOPs estimator: workload-aware | logical tokens/s: 75576.7 | valid tokens/s: 75576.7 | physical tokens/s: 77289.1 | loss tokens/s: 55794.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.413373E-01 | lm-global-supervised-tokens: 1.238581E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.596578E-02 | lm-token-buffer-mib: 2.336735E+02 | global_load_balancing_loss: 1.000401E+00 | loss scale: 1.0 | grad norm: 0.113 | params norm: 11172.390 | params rms: 0.060495 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:51:42.917881 | finish at 2026-10-17 18:33:57 + [2026-10-15 05:45:55] iteration 1398/ 2384 | consumed samples: 44736 | elapsed time per iteration (ms): 220808.3 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.39% | FLOPs estimator: workload-aware | logical tokens/s: 75980.9 | valid tokens/s: 75980.9 | physical tokens/s: 76364.7 | loss tokens/s: 54910.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.137965E-01 | lm-global-supervised-tokens: 1.212475E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.176763E-02 | lm-token-buffer-mib: 2.338250E+02 | global_load_balancing_loss: 9.993419E-01 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11172.352 | params rms: 0.060495 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:28:37.004505 | finish at 2026-10-17 18:14:32 + [2026-10-15 05:49:35] iteration 1399/ 2384 | consumed samples: 44768 | elapsed time per iteration (ms): 220487.0 | throughput per GPU (TFLOP/s/GPU): 25.2 | MFU 8.07% | FLOPs estimator: workload-aware | logical tokens/s: 76091.5 | valid tokens/s: 76091.5 | physical tokens/s: 76544.7 | loss tokens/s: 58243.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.415318E-01 | lm-global-supervised-tokens: 1.284190E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.490177E-02 | lm-token-buffer-mib: 2.330976E+02 | global_load_balancing_loss: 1.000636E+00 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11172.316 | params rms: 0.060494 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:19:39.695224 | finish at 2026-10-17 18:09:15 + [2026-10-15 05:53:39] iteration 1400/ 2384 | consumed samples: 44800 | elapsed time per iteration (ms): 243595.0 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.19% | FLOPs estimator: workload-aware | logical tokens/s: 68873.2 | valid tokens/s: 68873.2 | physical tokens/s: 69294.3 | loss tokens/s: 50496.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.892428E-01 | lm-global-supervised-tokens: 1.230062E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.541850E-02 | lm-token-buffer-mib: 2.328376E+02 | global_load_balancing_loss: 1.000977E+00 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11172.278 | params rms: 0.060494 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 18:34:57.480732 | finish at 2026-10-18 00:28:36 +times across ranks (ms): + save-checkpoint: + rank 0: 25016.79 + rank 1: 25016.84 + rank 2: 25016.82 + rank 3: 25016.82 + rank 4: 25016.69 + rank 5: 25016.82 + rank 6: 25016.80 + rank 7: 25016.79 + rank 8: 25014.98 + rank 9: 25015.02 + rank 10: 25014.97 + rank 11: 25014.94 + rank 12: 25014.96 + rank 13: 25014.94 + rank 14: 25014.96 + rank 15: 25014.95 + rank 16: 25015.50 + rank 17: 25015.49 + rank 18: 25015.52 + rank 19: 25015.50 + rank 20: 25015.52 + rank 21: 25015.48 + rank 22: 25015.48 + rank 23: 25015.46 + rank 24: 25015.65 + rank 25: 25015.65 + rank 26: 25015.65 + rank 27: 25015.58 + rank 28: 25015.56 + rank 29: 25016.52 + rank 30: 25015.65 + rank 31: 25015.59 + rank 32: 25015.46 + rank 33: 25015.49 + rank 34: 25015.40 + rank 35: 25015.45 + rank 36: 25015.42 + rank 37: 25015.41 + rank 38: 25015.34 + rank 39: 25015.35 + rank 40: 25015.46 + rank 41: 25015.49 + rank 42: 25015.46 + rank 43: 25015.47 + rank 44: 25015.45 + rank 45: 25015.44 + rank 46: 25015.43 + rank 47: 25015.47 + rank 48: 25014.95 + rank 49: 25014.99 + rank 50: 25014.84 + rank 51: 25015.01 + rank 52: 25015.07 + rank 53: 25015.09 + rank 54: 25014.92 + rank 55: 25014.95 + rank 56: 25015.46 + rank 57: 25015.47 + rank 58: 25015.48 + rank 59: 25015.48 + rank 60: 25015.47 + rank 61: 25015.46 + rank 62: 25015.44 + rank 63: 25015.44 + [2026-10-15 05:57:51] iteration 1401/ 2384 | consumed samples: 44832 | elapsed time per iteration (ms): 226873.1 | throughput per GPU (TFLOP/s/GPU): 23.7 | MFU 7.59% | FLOPs estimator: workload-aware | logical tokens/s: 73949.8 | valid tokens/s: 73949.8 | physical tokens/s: 74503.5 | loss tokens/s: 53614.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.665849E-01 | lm-global-supervised-tokens: 1.216363E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 2.118765E-01 | lm-token-buffer-mib: 2.335256E+02 | global_load_balancing_loss: 1.002159E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11172.238 | params rms: 0.060494 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:56:56.231561 | finish at 2026-10-17 19:54:47 + [2026-10-15 06:01:42] iteration 1402/ 2384 | consumed samples: 44864 | elapsed time per iteration (ms): 230799.6 | throughput per GPU (TFLOP/s/GPU): 24.0 | MFU 7.69% | FLOPs estimator: workload-aware | logical tokens/s: 72691.7 | valid tokens/s: 72691.7 | physical tokens/s: 73477.6 | loss tokens/s: 57242.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.673315E-01 | lm-global-supervised-tokens: 1.321164E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.540672E-02 | lm-token-buffer-mib: 2.358122E+02 | global_load_balancing_loss: 1.001511E+00 | loss scale: 1.0 | grad norm: 0.127 | params norm: 11172.199 | params rms: 0.060494 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:57:25.224648 | finish at 2026-10-17 20:59:07 + [2026-10-15 06:05:22] iteration 1403/ 2384 | consumed samples: 44896 | elapsed time per iteration (ms): 220206.9 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.03% | FLOPs estimator: workload-aware | logical tokens/s: 76188.3 | valid tokens/s: 76188.3 | physical tokens/s: 76955.7 | loss tokens/s: 59139.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.407120E-01 | lm-global-supervised-tokens: 1.302287E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.547882E-02 | lm-token-buffer-mib: 2.337328E+02 | global_load_balancing_loss: 9.998779E-01 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11172.161 | params rms: 0.060494 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:00:23.006907 | finish at 2026-10-17 18:05:45 + [2026-10-15 06:09:04] iteration 1404/ 2384 | consumed samples: 44928 | elapsed time per iteration (ms): 222468.6 | throughput per GPU (TFLOP/s/GPU): 24.4 | MFU 7.81% | FLOPs estimator: workload-aware | logical tokens/s: 75413.7 | valid tokens/s: 75413.7 | physical tokens/s: 76414.4 | loss tokens/s: 59490.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.320514E-01 | lm-global-supervised-tokens: 1.323478E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.325675E-02 | lm-token-buffer-mib: 2.353416E+02 | global_load_balancing_loss: 1.000211E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11172.125 | params rms: 0.060493 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:33:39.185276 | finish at 2026-10-17 18:42:44 + [2026-10-15 06:12:47] iteration 1405/ 2384 | consumed samples: 44960 | elapsed time per iteration (ms): 222875.9 | throughput per GPU (TFLOP/s/GPU): 24.2 | MFU 7.74% | FLOPs estimator: workload-aware | logical tokens/s: 75275.7 | valid tokens/s: 75275.7 | physical tokens/s: 76843.0 | loss tokens/s: 62266.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.437047E-01 | lm-global-supervised-tokens: 1.387766E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.760777E-02 | lm-token-buffer-mib: 2.365432E+02 | global_load_balancing_loss: 9.996295E-01 | loss scale: 1.0 | grad norm: 0.162 | params norm: 11172.089 | params rms: 0.060493 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:36:35.548144 | finish at 2026-10-17 18:49:23 + [2026-10-15 06:16:28] iteration 1406/ 2384 | consumed samples: 44992 | elapsed time per iteration (ms): 220651.7 | throughput per GPU (TFLOP/s/GPU): 27.0 | MFU 8.66% | FLOPs estimator: workload-aware | logical tokens/s: 76034.7 | valid tokens/s: 76034.7 | physical tokens/s: 76370.1 | loss tokens/s: 55754.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.895839E-01 | lm-global-supervised-tokens: 1.230236E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.682912E-02 | lm-token-buffer-mib: 2.324194E+02 | global_load_balancing_loss: 9.988178E-01 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11172.055 | params rms: 0.060493 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:56:37.374278 | finish at 2026-10-17 18:13:05 + [2026-10-15 06:20:11] iteration 1407/ 2384 | consumed samples: 45024 | elapsed time per iteration (ms): 222783.9 | throughput per GPU (TFLOP/s/GPU): 23.8 | MFU 7.62% | FLOPs estimator: workload-aware | logical tokens/s: 75307.1 | valid tokens/s: 75307.1 | physical tokens/s: 75996.9 | loss tokens/s: 55089.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.220562E-01 | lm-global-supervised-tokens: 1.227297E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.583567E-02 | lm-token-buffer-mib: 2.347871E+02 | global_load_balancing_loss: 1.000301E+00 | loss scale: 1.0 | grad norm: 0.103 | params norm: 11172.018 | params rms: 0.060493 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:27:39.868691 | finish at 2026-10-17 18:47:51 + [2026-10-15 06:23:57] iteration 1408/ 2384 | consumed samples: 45056 | elapsed time per iteration (ms): 226601.8 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.90% | FLOPs estimator: workload-aware | logical tokens/s: 74038.2 | valid tokens/s: 74038.2 | physical tokens/s: 75709.8 | loss tokens/s: 57170.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.176404E-01 | lm-global-supervised-tokens: 1.295497E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.913245E-02 | lm-token-buffer-mib: 2.392335E+02 | global_load_balancing_loss: 1.000359E+00 | loss scale: 1.0 | grad norm: 0.211 | params norm: 11171.981 | params rms: 0.060493 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:26:03.343037 | finish at 2026-10-17 19:50:01 + [2026-10-15 06:27:40] iteration 1409/ 2384 | consumed samples: 45088 | elapsed time per iteration (ms): 223113.1 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.94% | FLOPs estimator: workload-aware | logical tokens/s: 75195.9 | valid tokens/s: 75195.9 | physical tokens/s: 76100.7 | loss tokens/s: 55400.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.575749E-01 | lm-global-supervised-tokens: 1.236057E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.228997E-02 | lm-token-buffer-mib: 2.341277E+02 | global_load_balancing_loss: 1.001781E+00 | loss scale: 1.0 | grad norm: 0.186 | params norm: 11171.946 | params rms: 0.060492 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:25:35.293007 | finish at 2026-10-17 18:53:16 + [2026-10-15 06:31:46] iteration 1410/ 2384 | consumed samples: 45120 | elapsed time per iteration (ms): 245559.8 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.68% | FLOPs estimator: workload-aware | logical tokens/s: 68322.1 | valid tokens/s: 68322.1 | physical tokens/s: 68994.0 | loss tokens/s: 57657.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.203190E-01 | lm-global-supervised-tokens: 1.415835E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.877526E-02 | lm-token-buffer-mib: 2.334127E+02 | global_load_balancing_loss: 1.000861E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11171.911 | params rms: 0.060492 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 18:26:15.288305 | finish at 2026-10-18 00:58:01 + [2026-10-15 06:35:28] iteration 1411/ 2384 | consumed samples: 45152 | elapsed time per iteration (ms): 221933.8 | throughput per GPU (TFLOP/s/GPU): 23.8 | MFU 7.64% | FLOPs estimator: workload-aware | logical tokens/s: 75595.4 | valid tokens/s: 75595.4 | physical tokens/s: 75864.9 | loss tokens/s: 57224.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.723897E-01 | lm-global-supervised-tokens: 1.270009E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.459318E-02 | lm-token-buffer-mib: 2.322064E+02 | global_load_balancing_loss: 1.000565E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11171.875 | params rms: 0.060492 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:59:01.550498 | finish at 2026-10-17 18:34:29 + [2026-10-15 06:39:12] iteration 1412/ 2384 | consumed samples: 45184 | elapsed time per iteration (ms): 224561.3 | throughput per GPU (TFLOP/s/GPU): 23.5 | MFU 7.54% | FLOPs estimator: workload-aware | logical tokens/s: 74710.9 | valid tokens/s: 74710.9 | physical tokens/s: 75840.8 | loss tokens/s: 60341.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.179404E-01 | lm-global-supervised-tokens: 1.355024E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.934748E-02 | lm-token-buffer-mib: 2.341074E+02 | global_load_balancing_loss: 1.000597E+00 | loss scale: 1.0 | grad norm: 0.126 | params norm: 11171.840 | params rms: 0.060492 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:37:53.538912 | finish at 2026-10-17 19:17:06 + [2026-10-15 06:42:52] iteration 1413/ 2384 | consumed samples: 45216 | elapsed time per iteration (ms): 219524.0 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.31% | FLOPs estimator: workload-aware | logical tokens/s: 76425.3 | valid tokens/s: 76425.3 | physical tokens/s: 77223.1 | loss tokens/s: 57770.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.385349E-01 | lm-global-supervised-tokens: 1.268194E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.396981E-02 | lm-token-buffer-mib: 2.329583E+02 | global_load_balancing_loss: 9.998805E-01 | loss scale: 1.0 | grad norm: 0.135 | params norm: 11171.803 | params rms: 0.060492 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:12:37.842361 | finish at 2026-10-17 17:55:30 + [2026-10-15 06:46:38] iteration 1414/ 2384 | consumed samples: 45248 | elapsed time per iteration (ms): 225694.6 | throughput per GPU (TFLOP/s/GPU): 25.5 | MFU 8.19% | FLOPs estimator: workload-aware | logical tokens/s: 74335.9 | valid tokens/s: 74335.9 | physical tokens/s: 74662.0 | loss tokens/s: 56197.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.424463E-01 | lm-global-supervised-tokens: 1.268356E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.514129E-02 | lm-token-buffer-mib: 2.330014E+02 | global_load_balancing_loss: 1.000254E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11171.765 | params rms: 0.060491 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:48:43.734350 | finish at 2026-10-17 19:35:21 + [2026-10-15 06:50:22] iteration 1415/ 2384 | consumed samples: 45280 | elapsed time per iteration (ms): 224523.1 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.05% | FLOPs estimator: workload-aware | logical tokens/s: 74723.8 | valid tokens/s: 74723.8 | physical tokens/s: 75160.5 | loss tokens/s: 58406.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.036171E-01 | lm-global-supervised-tokens: 1.311362E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.521606E-02 | lm-token-buffer-mib: 2.335874E+02 | global_load_balancing_loss: 1.000843E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11171.726 | params rms: 0.060491 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:26:02.908522 | finish at 2026-10-17 19:16:25 + [2026-10-15 06:54:05] iteration 1416/ 2384 | consumed samples: 45312 | elapsed time per iteration (ms): 223249.3 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.88% | FLOPs estimator: workload-aware | logical tokens/s: 75150.0 | valid tokens/s: 75150.0 | physical tokens/s: 75669.9 | loss tokens/s: 57325.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.159119E-01 | lm-global-supervised-tokens: 1.279780E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.178737E-02 | lm-token-buffer-mib: 2.339692E+02 | global_load_balancing_loss: 1.000187E+00 | loss scale: 1.0 | grad norm: 0.142 | params norm: 11171.689 | params rms: 0.060491 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:01:45.301170 | finish at 2026-10-17 18:55:51 + [2026-10-15 06:57:45] iteration 1417/ 2384 | consumed samples: 45344 | elapsed time per iteration (ms): 219341.5 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.40% | FLOPs estimator: workload-aware | logical tokens/s: 76488.8 | valid tokens/s: 76488.8 | physical tokens/s: 76910.9 | loss tokens/s: 53380.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.412692E-01 | lm-global-supervised-tokens: 1.170858E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.664730E-02 | lm-token-buffer-mib: 2.322073E+02 | global_load_balancing_loss: 9.999983E-01 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11171.652 | params rms: 0.060491 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:55:03.197804 | finish at 2026-10-17 17:52:48 + [2026-10-15 07:01:27] iteration 1418/ 2384 | consumed samples: 45376 | elapsed time per iteration (ms): 222349.4 | throughput per GPU (TFLOP/s/GPU): 24.9 | MFU 8.00% | FLOPs estimator: workload-aware | logical tokens/s: 75454.1 | valid tokens/s: 75454.1 | physical tokens/s: 75828.8 | loss tokens/s: 59550.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.230268E-01 | lm-global-supervised-tokens: 1.324102E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.670488E-02 | lm-token-buffer-mib: 2.334548E+02 | global_load_balancing_loss: 9.997487E-01 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11171.614 | params rms: 0.060491 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:39:49.493265 | finish at 2026-10-17 18:41:17 + [2026-10-15 07:05:06] iteration 1419/ 2384 | consumed samples: 45408 | elapsed time per iteration (ms): 219206.2 | throughput per GPU (TFLOP/s/GPU): 23.7 | MFU 7.59% | FLOPs estimator: workload-aware | logical tokens/s: 76536.1 | valid tokens/s: 76536.1 | physical tokens/s: 77379.1 | loss tokens/s: 57947.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.967037E-01 | lm-global-supervised-tokens: 1.270240E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.589029E-02 | lm-token-buffer-mib: 2.326736E+02 | global_load_balancing_loss: 1.000644E+00 | loss scale: 1.0 | grad norm: 0.275 | params norm: 11171.577 | params rms: 0.060490 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:45:34.019930 | finish at 2026-10-17 17:50:40 + [2026-10-15 07:09:10] iteration 1420/ 2384 | consumed samples: 45440 | elapsed time per iteration (ms): 243228.0 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.55% | FLOPs estimator: workload-aware | logical tokens/s: 68977.1 | valid tokens/s: 68977.1 | physical tokens/s: 69151.0 | loss tokens/s: 53482.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.154421E-01 | lm-global-supervised-tokens: 1.300838E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.340451E-02 | lm-token-buffer-mib: 2.328017E+02 | global_load_balancing_loss: 1.000590E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11171.535 | params rms: 0.060490 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:07:51.745502 | finish at 2026-10-18 00:17:01 + [2026-10-15 07:12:53] iteration 1421/ 2384 | consumed samples: 45472 | elapsed time per iteration (ms): 223945.0 | throughput per GPU (TFLOP/s/GPU): 18.5 | MFU 5.94% | FLOPs estimator: workload-aware | logical tokens/s: 74916.5 | valid tokens/s: 74916.5 | physical tokens/s: 75420.2 | loss tokens/s: 53257.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.142800E-01 | lm-global-supervised-tokens: 1.192676E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.708045E-02 | lm-token-buffer-mib: 2.338353E+02 | global_load_balancing_loss: 1.000236E+00 | loss scale: 1.0 | grad norm: 0.176 | params norm: 11171.496 | params rms: 0.060490 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:54:19.012665 | finish at 2026-10-17 19:07:12 + [2026-10-15 07:16:33] iteration 1422/ 2384 | consumed samples: 45504 | elapsed time per iteration (ms): 219634.5 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.55% | FLOPs estimator: workload-aware | logical tokens/s: 76386.9 | valid tokens/s: 76386.9 | physical tokens/s: 76812.4 | loss tokens/s: 58975.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.608705E-01 | lm-global-supervised-tokens: 1.295312E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.286944E-02 | lm-token-buffer-mib: 2.329200E+02 | global_load_balancing_loss: 1.001948E+00 | loss scale: 1.0 | grad norm: 0.189 | params norm: 11171.460 | params rms: 0.060490 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:41:28.413338 | finish at 2026-10-17 17:58:02 + [2026-10-15 07:20:13] iteration 1423/ 2384 | consumed samples: 45536 | elapsed time per iteration (ms): 220004.6 | throughput per GPU (TFLOP/s/GPU): 23.8 | MFU 7.61% | FLOPs estimator: workload-aware | logical tokens/s: 76258.4 | valid tokens/s: 76258.4 | physical tokens/s: 76513.0 | loss tokens/s: 55471.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.617047E-01 | lm-global-supervised-tokens: 1.220398E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.493610E-02 | lm-token-buffer-mib: 2.332171E+02 | global_load_balancing_loss: 9.992663E-01 | loss scale: 1.0 | grad norm: 0.110 | params norm: 11171.424 | params rms: 0.060490 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:43:44.377801 | finish at 2026-10-17 18:03:58 + [2026-10-15 07:23:55] iteration 1424/ 2384 | consumed samples: 45568 | elapsed time per iteration (ms): 221793.5 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.91% | FLOPs estimator: workload-aware | logical tokens/s: 75643.4 | valid tokens/s: 75643.3 | physical tokens/s: 76058.3 | loss tokens/s: 53513.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.479492E-01 | lm-global-supervised-tokens: 1.186888E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.909878E-02 | lm-token-buffer-mib: 2.329945E+02 | global_load_balancing_loss: 1.000448E+00 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11171.386 | params rms: 0.060489 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:08:41.766129 | finish at 2026-10-17 18:32:37 + [2026-10-15 07:27:39] iteration 1425/ 2384 | consumed samples: 45600 | elapsed time per iteration (ms): 224382.4 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.59% | FLOPs estimator: workload-aware | logical tokens/s: 74770.6 | valid tokens/s: 74770.6 | physical tokens/s: 75257.8 | loss tokens/s: 59370.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.954720E-01 | lm-global-supervised-tokens: 1.332162E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.230757E-02 | lm-token-buffer-mib: 2.339652E+02 | global_load_balancing_loss: 9.975259E-01 | loss scale: 1.0 | grad norm: 0.322 | params norm: 11171.353 | params rms: 0.060489 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:46:22.703343 | finish at 2026-10-17 19:14:02 + [2026-10-15 07:31:20] iteration 1426/ 2384 | consumed samples: 45632 | elapsed time per iteration (ms): 220857.5 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.60% | FLOPs estimator: workload-aware | logical tokens/s: 75963.9 | valid tokens/s: 75963.9 | physical tokens/s: 76646.4 | loss tokens/s: 59446.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.453922E-01 | lm-global-supervised-tokens: 1.312926E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.404922E-02 | lm-token-buffer-mib: 2.334970E+02 | global_load_balancing_loss: 9.998701E-01 | loss scale: 1.0 | grad norm: 0.102 | params norm: 11171.318 | params rms: 0.060489 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:46:21.464517 | finish at 2026-10-17 18:17:42 + [2026-10-15 07:35:00] iteration 1427/ 2384 | consumed samples: 45664 | elapsed time per iteration (ms): 219955.5 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.92% | FLOPs estimator: workload-aware | logical tokens/s: 76275.3 | valid tokens/s: 76275.3 | physical tokens/s: 76875.8 | loss tokens/s: 56036.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.096875E-01 | lm-global-supervised-tokens: 1.232560E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.457852E-02 | lm-token-buffer-mib: 2.338334E+02 | global_load_balancing_loss: 1.001202E+00 | loss scale: 1.0 | grad norm: 0.184 | params norm: 11171.282 | params rms: 0.060489 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:28:17.460395 | finish at 2026-10-17 18:03:18 + [2026-10-15 07:38:46] iteration 1428/ 2384 | consumed samples: 45696 | elapsed time per iteration (ms): 225491.1 | throughput per GPU (TFLOP/s/GPU): 19.2 | MFU 6.16% | FLOPs estimator: workload-aware | logical tokens/s: 74402.9 | valid tokens/s: 74402.8 | physical tokens/s: 76010.6 | loss tokens/s: 57184.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.232827E-01 | lm-global-supervised-tokens: 1.289449E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.389642E-02 | lm-token-buffer-mib: 2.384813E+02 | global_load_balancing_loss: 1.001051E+00 | loss scale: 1.0 | grad norm: 0.180 | params norm: 11171.244 | params rms: 0.060489 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:52:49.535660 | finish at 2026-10-17 19:31:35 + [2026-10-15 07:42:32] iteration 1429/ 2384 | consumed samples: 45728 | elapsed time per iteration (ms): 226653.8 | throughput per GPU (TFLOP/s/GPU): 19.3 | MFU 6.17% | FLOPs estimator: workload-aware | logical tokens/s: 74021.3 | valid tokens/s: 74021.3 | physical tokens/s: 74430.5 | loss tokens/s: 56613.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.652009E-01 | lm-global-supervised-tokens: 1.283163E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.995861E-02 | lm-token-buffer-mib: 2.332971E+02 | global_load_balancing_loss: 1.000824E+00 | loss scale: 1.0 | grad norm: 0.032 | params norm: 11171.204 | params rms: 0.060488 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:07:34.403601 | finish at 2026-10-17 19:50:07 + [2026-10-15 07:46:37] iteration 1430/ 2384 | consumed samples: 45760 | elapsed time per iteration (ms): 244670.2 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.71% | FLOPs estimator: workload-aware | logical tokens/s: 68570.5 | valid tokens/s: 68570.5 | physical tokens/s: 69021.4 | loss tokens/s: 51341.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.713993E-01 | lm-global-supervised-tokens: 1.256161E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.179330E-02 | lm-token-buffer-mib: 2.325956E+02 | global_load_balancing_loss: 1.002154E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11171.164 | params rms: 0.060488 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:50:15.393877 | finish at 2026-10-18 00:36:52 + [2026-10-15 07:50:21] iteration 1431/ 2384 | consumed samples: 45792 | elapsed time per iteration (ms): 223956.1 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.27% | FLOPs estimator: workload-aware | logical tokens/s: 74912.9 | valid tokens/s: 74912.9 | physical tokens/s: 75329.9 | loss tokens/s: 58025.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.846085E-01 | lm-global-supervised-tokens: 1.299518E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.836875E-02 | lm-token-buffer-mib: 2.338191E+02 | global_load_balancing_loss: 1.001536E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11171.124 | params rms: 0.060488 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:17:10.130569 | finish at 2026-10-17 19:07:31 + [2026-10-15 07:54:02] iteration 1432/ 2384 | consumed samples: 45824 | elapsed time per iteration (ms): 220956.3 | throughput per GPU (TFLOP/s/GPU): 24.3 | MFU 7.77% | FLOPs estimator: workload-aware | logical tokens/s: 75929.8 | valid tokens/s: 75929.8 | physical tokens/s: 76281.1 | loss tokens/s: 53614.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.213872E-01 | lm-global-supervised-tokens: 1.184656E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.605138E-02 | lm-token-buffer-mib: 2.331476E+02 | global_load_balancing_loss: 1.001498E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11171.086 | params rms: 0.060488 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:25:50.428488 | finish at 2026-10-17 18:19:52 + [2026-10-15 07:57:43] iteration 1433/ 2384 | consumed samples: 45856 | elapsed time per iteration (ms): 221105.5 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.26% | FLOPs estimator: workload-aware | logical tokens/s: 75878.5 | valid tokens/s: 75878.4 | physical tokens/s: 76475.9 | loss tokens/s: 53260.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.370202E-01 | lm-global-supervised-tokens: 1.177626E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.695765E-02 | lm-token-buffer-mib: 2.340056E+02 | global_load_balancing_loss: 9.984616E-01 | loss scale: 1.0 | grad norm: 0.187 | params norm: 11171.048 | params rms: 0.060488 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:24:31.344768 | finish at 2026-10-17 18:22:14 + [2026-10-15 08:01:23] iteration 1434/ 2384 | consumed samples: 45888 | elapsed time per iteration (ms): 220043.2 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.73% | FLOPs estimator: workload-aware | logical tokens/s: 76245.0 | valid tokens/s: 76245.0 | physical tokens/s: 77153.4 | loss tokens/s: 51301.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.059245E-01 | lm-global-supervised-tokens: 1.128859E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.935568E-02 | lm-token-buffer-mib: 2.339932E+02 | global_load_balancing_loss: 9.988056E-01 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11171.010 | params rms: 0.060487 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:04:01.067874 | finish at 2026-10-17 18:05:24 + [2026-10-15 08:05:04] iteration 1435/ 2384 | consumed samples: 45920 | elapsed time per iteration (ms): 220915.5 | throughput per GPU (TFLOP/s/GPU): 20.2 | MFU 6.46% | FLOPs estimator: workload-aware | logical tokens/s: 75944.0 | valid tokens/s: 75944.0 | physical tokens/s: 76865.0 | loss tokens/s: 59384.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.229426E-01 | lm-global-supervised-tokens: 1.311898E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.667013E-02 | lm-token-buffer-mib: 2.354863E+02 | global_load_balancing_loss: 9.981475E-01 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11170.973 | params rms: 0.060487 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:14:08.852871 | finish at 2026-10-17 18:19:13 + [2026-10-15 08:08:51] iteration 1436/ 2384 | consumed samples: 45952 | elapsed time per iteration (ms): 226844.3 | throughput per GPU (TFLOP/s/GPU): 26.5 | MFU 8.48% | FLOPs estimator: workload-aware | logical tokens/s: 73959.0 | valid tokens/s: 73959.0 | physical tokens/s: 74673.2 | loss tokens/s: 56036.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.014760E-01 | lm-global-supervised-tokens: 1.271166E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.268311E-02 | lm-token-buffer-mib: 2.356450E+02 | global_load_balancing_loss: 9.991318E-01 | loss scale: 1.0 | grad norm: 0.122 | params norm: 11170.935 | params rms: 0.060487 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:44:08.402533 | finish at 2026-10-17 19:52:59 + [2026-10-15 08:12:33] iteration 1437/ 2384 | consumed samples: 45984 | elapsed time per iteration (ms): 221786.7 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.33% | FLOPs estimator: workload-aware | logical tokens/s: 75645.7 | valid tokens/s: 75645.7 | physical tokens/s: 76180.7 | loss tokens/s: 56229.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.954725E-01 | lm-global-supervised-tokens: 1.247092E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.459732E-02 | lm-token-buffer-mib: 2.340122E+02 | global_load_balancing_loss: 9.998967E-01 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11170.894 | params rms: 0.060487 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:20:31.964269 | finish at 2026-10-17 18:33:05 + [2026-10-15 08:16:18] iteration 1438/ 2384 | consumed samples: 46016 | elapsed time per iteration (ms): 225179.2 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.25% | FLOPs estimator: workload-aware | logical tokens/s: 74505.9 | valid tokens/s: 74505.8 | physical tokens/s: 75612.8 | loss tokens/s: 58015.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.975650E-01 | lm-global-supervised-tokens: 1.306393E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.396286E-02 | lm-token-buffer-mib: 2.370179E+02 | global_load_balancing_loss: 1.001556E+00 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11170.850 | params rms: 0.060487 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:10:19.512988 | finish at 2026-10-17 19:26:37 + [2026-10-15 08:20:01] iteration 1439/ 2384 | consumed samples: 46048 | elapsed time per iteration (ms): 223538.7 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.34% | FLOPs estimator: workload-aware | logical tokens/s: 75052.8 | valid tokens/s: 75052.8 | physical tokens/s: 75974.7 | loss tokens/s: 60832.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.025442E-01 | lm-global-supervised-tokens: 1.359843E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.637525E-02 | lm-token-buffer-mib: 2.334921E+02 | global_load_balancing_loss: 1.001342E+00 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11170.809 | params rms: 0.060486 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:40:44.075203 | finish at 2026-10-17 19:00:45 + [2026-10-15 08:24:05] iteration 1440/ 2384 | consumed samples: 46080 | elapsed time per iteration (ms): 243902.1 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.06% | FLOPs estimator: workload-aware | logical tokens/s: 68786.6 | valid tokens/s: 68786.6 | physical tokens/s: 69294.2 | loss tokens/s: 52369.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.399855E-01 | lm-global-supervised-tokens: 1.277300E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.877447E-02 | lm-token-buffer-mib: 2.344464E+02 | global_load_balancing_loss: 1.002984E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11170.771 | params rms: 0.060486 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:57:23.572578 | finish at 2026-10-18 00:21:29 +times across ranks (ms): + save-checkpoint: + rank 0: 24919.37 + rank 1: 24919.33 + rank 2: 24919.38 + rank 3: 24919.37 + rank 4: 24919.34 + rank 5: 24919.36 + rank 6: 24919.36 + rank 7: 24919.35 + rank 8: 24918.83 + rank 9: 24918.84 + rank 10: 24918.86 + rank 11: 24918.84 + rank 12: 24918.85 + rank 13: 24918.83 + rank 14: 24918.85 + rank 15: 24918.83 + rank 16: 24919.10 + rank 17: 24919.10 + rank 18: 24919.10 + rank 19: 24919.09 + rank 20: 24919.06 + rank 21: 24919.06 + rank 22: 24918.97 + rank 23: 24919.04 + rank 24: 24919.06 + rank 25: 24919.02 + rank 26: 24919.13 + rank 27: 24919.05 + rank 28: 24918.97 + rank 29: 24919.08 + rank 30: 24918.98 + rank 31: 24919.01 + rank 32: 24918.29 + rank 33: 24918.29 + rank 34: 24918.43 + rank 35: 24918.47 + rank 36: 24918.35 + rank 37: 24918.40 + rank 38: 24918.35 + rank 39: 24918.34 + rank 40: 24918.71 + rank 41: 24918.72 + rank 42: 24918.72 + rank 43: 24918.74 + rank 44: 24918.75 + rank 45: 24918.73 + rank 46: 24918.72 + rank 47: 24918.72 + rank 48: 24918.78 + rank 49: 24918.84 + rank 50: 24918.83 + rank 51: 24918.77 + rank 52: 24918.79 + rank 53: 24919.02 + rank 54: 24918.76 + rank 55: 24918.84 + rank 56: 24920.27 + rank 57: 24920.30 + rank 58: 24920.32 + rank 59: 24920.16 + rank 60: 24920.30 + rank 61: 24920.31 + rank 62: 24920.26 + rank 63: 24920.30 + [2026-10-15 08:28:23] iteration 1441/ 2384 | consumed samples: 46112 | elapsed time per iteration (ms): 232763.9 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.88% | FLOPs estimator: workload-aware | logical tokens/s: 72078.2 | valid tokens/s: 72078.2 | physical tokens/s: 72539.3 | loss tokens/s: 54981.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.622444E-01 | lm-global-supervised-tokens: 1.279767E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 6.146071E-01 | lm-token-buffer-mib: 2.342235E+02 | global_load_balancing_loss: 1.002879E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11170.735 | params rms: 0.060486 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:58:16.393937 | finish at 2026-10-17 21:26:39 + [2026-10-15 08:32:14] iteration 1442/ 2384 | consumed samples: 46144 | elapsed time per iteration (ms): 231416.9 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.64% | FLOPs estimator: workload-aware | logical tokens/s: 72497.7 | valid tokens/s: 72497.7 | physical tokens/s: 73399.9 | loss tokens/s: 58733.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.014608E-01 | lm-global-supervised-tokens: 1.359193E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.295757E-02 | lm-token-buffer-mib: 2.352914E+02 | global_load_balancing_loss: 1.000380E+00 | loss scale: 1.0 | grad norm: 0.128 | params norm: 11170.700 | params rms: 0.060486 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:33:14.686036 | finish at 2026-10-17 21:05:29 + [2026-10-15 08:35:56] iteration 1443/ 2384 | consumed samples: 46176 | elapsed time per iteration (ms): 221863.8 | throughput per GPU (TFLOP/s/GPU): 20.2 | MFU 6.48% | FLOPs estimator: workload-aware | logical tokens/s: 75619.4 | valid tokens/s: 75619.4 | physical tokens/s: 76509.6 | loss tokens/s: 58189.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.943210E-01 | lm-global-supervised-tokens: 1.291007E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.300075E-02 | lm-token-buffer-mib: 2.346780E+02 | global_load_balancing_loss: 9.985027E-01 | loss scale: 1.0 | grad norm: 0.124 | params norm: 11170.669 | params rms: 0.060486 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:59:33.840109 | finish at 2026-10-17 18:35:30 + [2026-10-15 08:39:39] iteration 1444/ 2384 | consumed samples: 46208 | elapsed time per iteration (ms): 222267.7 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.33% | FLOPs estimator: workload-aware | logical tokens/s: 75482.0 | valid tokens/s: 75482.0 | physical tokens/s: 76086.7 | loss tokens/s: 58432.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.945305E-01 | lm-global-supervised-tokens: 1.298761E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.173872E-02 | lm-token-buffer-mib: 2.340530E+02 | global_load_balancing_loss: 9.978081E-01 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11170.636 | params rms: 0.060485 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:02:11.592016 | finish at 2026-10-17 18:41:50 + [2026-10-15 08:43:19] iteration 1445/ 2384 | consumed samples: 46240 | elapsed time per iteration (ms): 220861.5 | throughput per GPU (TFLOP/s/GPU): 19.6 | MFU 6.30% | FLOPs estimator: workload-aware | logical tokens/s: 75962.5 | valid tokens/s: 75962.5 | physical tokens/s: 77055.9 | loss tokens/s: 56052.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.930296E-01 | lm-global-supervised-tokens: 1.237992E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.883465E-02 | lm-token-buffer-mib: 2.348162E+02 | global_load_balancing_loss: 9.978796E-01 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11170.602 | params rms: 0.060485 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:36:28.982105 | finish at 2026-10-17 18:19:48 + [2026-10-15 08:47:04] iteration 1446/ 2384 | consumed samples: 46272 | elapsed time per iteration (ms): 224219.8 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.66% | FLOPs estimator: workload-aware | logical tokens/s: 74824.8 | valid tokens/s: 74824.8 | physical tokens/s: 75635.8 | loss tokens/s: 57575.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.407272E-01 | lm-global-supervised-tokens: 1.290956E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.361590E-02 | lm-token-buffer-mib: 2.353054E+02 | global_load_balancing_loss: 9.982684E-01 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11170.568 | params rms: 0.060485 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:25:18.132812 | finish at 2026-10-17 19:12:22 + [2026-10-15 08:50:44] iteration 1447/ 2384 | consumed samples: 46304 | elapsed time per iteration (ms): 220364.4 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.41% | FLOPs estimator: workload-aware | logical tokens/s: 76133.9 | valid tokens/s: 76133.9 | physical tokens/s: 77689.2 | loss tokens/s: 58059.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.577459E-01 | lm-global-supervised-tokens: 1.279426E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.482985E-02 | lm-token-buffer-mib: 2.340254E+02 | global_load_balancing_loss: 1.000747E+00 | loss scale: 1.0 | grad norm: 0.152 | params norm: 11170.534 | params rms: 0.060485 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:21:21.452992 | finish at 2026-10-17 18:12:05 + [2026-10-15 08:54:27] iteration 1448/ 2384 | consumed samples: 46336 | elapsed time per iteration (ms): 223383.9 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.63% | FLOPs estimator: workload-aware | logical tokens/s: 75104.8 | valid tokens/s: 75104.8 | physical tokens/s: 75862.1 | loss tokens/s: 57021.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.972372E-01 | lm-global-supervised-tokens: 1.273767E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.614167E-02 | lm-token-buffer-mib: 2.354865E+02 | global_load_balancing_loss: 1.002102E+00 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11170.502 | params rms: 0.060485 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:04:47.345060 | finish at 2026-10-17 18:59:15 + [2026-10-15 08:58:09] iteration 1449/ 2384 | consumed samples: 46368 | elapsed time per iteration (ms): 221124.4 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.36% | FLOPs estimator: workload-aware | logical tokens/s: 75871.9 | valid tokens/s: 75871.9 | physical tokens/s: 76859.5 | loss tokens/s: 60671.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.774756E-01 | lm-global-supervised-tokens: 1.341585E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.295573E-02 | lm-token-buffer-mib: 2.358106E+02 | global_load_balancing_loss: 1.002942E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11170.474 | params rms: 0.060484 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:25:51.309226 | finish at 2026-10-17 18:24:00 + [2026-10-15 09:02:13] iteration 1450/ 2384 | consumed samples: 46400 | elapsed time per iteration (ms): 244009.6 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.69% | FLOPs estimator: workload-aware | logical tokens/s: 68756.3 | valid tokens/s: 68756.3 | physical tokens/s: 69385.3 | loss tokens/s: 50257.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.950789E-01 | lm-global-supervised-tokens: 1.226326E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.578554E-02 | lm-token-buffer-mib: 2.335272E+02 | global_load_balancing_loss: 1.002553E+00 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11170.446 | params rms: 0.060484 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:18:24.962098 | finish at 2026-10-18 00:20:37 + [2026-10-15 09:05:56] iteration 1451/ 2384 | consumed samples: 46432 | elapsed time per iteration (ms): 223385.4 | throughput per GPU (TFLOP/s/GPU): 18.7 | MFU 5.99% | FLOPs estimator: workload-aware | logical tokens/s: 75104.3 | valid tokens/s: 75104.3 | physical tokens/s: 75954.1 | loss tokens/s: 57999.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.455409E-01 | lm-global-supervised-tokens: 1.295623E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.374863E-02 | lm-token-buffer-mib: 2.357766E+02 | global_load_balancing_loss: 1.000619E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11170.414 | params rms: 0.060484 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:53:38.546888 | finish at 2026-10-17 18:59:34 + [2026-10-15 09:09:38] iteration 1452/ 2384 | consumed samples: 46464 | elapsed time per iteration (ms): 221782.9 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.56% | FLOPs estimator: workload-aware | logical tokens/s: 75646.8 | valid tokens/s: 75646.8 | physical tokens/s: 75955.2 | loss tokens/s: 58651.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.786160E-01 | lm-global-supervised-tokens: 1.300787E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.151722E-02 | lm-token-buffer-mib: 2.326694E+02 | global_load_balancing_loss: 9.992835E-01 | loss scale: 1.0 | grad norm: 0.518 | params norm: 11170.377 | params rms: 0.060484 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:25:01.661111 | finish at 2026-10-17 18:34:39 + [2026-10-15 09:13:24] iteration 1453/ 2384 | consumed samples: 46496 | elapsed time per iteration (ms): 226769.8 | throughput per GPU (TFLOP/s/GPU): 23.4 | MFU 7.51% | FLOPs estimator: workload-aware | logical tokens/s: 73983.3 | valid tokens/s: 73983.3 | physical tokens/s: 75685.9 | loss tokens/s: 60054.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.728504E-01 | lm-global-supervised-tokens: 1.361861E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.441850E-02 | lm-token-buffer-mib: 2.400426E+02 | global_load_balancing_loss: 9.994856E-01 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11170.326 | params rms: 0.060484 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:38:42.641134 | finish at 2026-10-17 19:52:07 + [2026-10-15 09:17:09] iteration 1454/ 2384 | consumed samples: 46528 | elapsed time per iteration (ms): 224436.5 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.20% | FLOPs estimator: workload-aware | logical tokens/s: 74752.5 | valid tokens/s: 74752.5 | physical tokens/s: 75276.2 | loss tokens/s: 55438.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.310674E-01 | lm-global-supervised-tokens: 1.244245E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.395857E-02 | lm-token-buffer-mib: 2.339264E+02 | global_load_balancing_loss: 1.000050E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11170.287 | params rms: 0.060483 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:58:45.965910 | finish at 2026-10-17 19:15:55 + [2026-10-15 09:20:54] iteration 1455/ 2384 | consumed samples: 46560 | elapsed time per iteration (ms): 225339.9 | throughput per GPU (TFLOP/s/GPU): 19.6 | MFU 6.27% | FLOPs estimator: workload-aware | logical tokens/s: 74452.8 | valid tokens/s: 74452.8 | physical tokens/s: 75296.7 | loss tokens/s: 56604.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.877314E-01 | lm-global-supervised-tokens: 1.275513E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.417360E-02 | lm-token-buffer-mib: 2.346468E+02 | global_load_balancing_loss: 1.000398E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11170.256 | params rms: 0.060483 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:09:00.787714 | finish at 2026-10-17 19:29:55 + [2026-10-15 09:24:38] iteration 1456/ 2384 | consumed samples: 46592 | elapsed time per iteration (ms): 223659.8 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.65% | FLOPs estimator: workload-aware | logical tokens/s: 75012.0 | valid tokens/s: 75012.0 | physical tokens/s: 75535.3 | loss tokens/s: 58096.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.759669E-01 | lm-global-supervised-tokens: 1.299386E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.246773E-02 | lm-token-buffer-mib: 2.345433E+02 | global_load_balancing_loss: 9.998999E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11170.233 | params rms: 0.060483 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:39:16.333168 | finish at 2026-10-17 19:03:54 + [2026-10-15 09:28:20] iteration 1457/ 2384 | consumed samples: 46624 | elapsed time per iteration (ms): 222542.0 | throughput per GPU (TFLOP/s/GPU): 19.3 | MFU 6.18% | FLOPs estimator: workload-aware | logical tokens/s: 75389.0 | valid tokens/s: 75389.0 | physical tokens/s: 76473.2 | loss tokens/s: 55270.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.395988E-01 | lm-global-supervised-tokens: 1.229993E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.804596E-02 | lm-token-buffer-mib: 2.358770E+02 | global_load_balancing_loss: 1.000613E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11170.214 | params rms: 0.060483 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:18:16.407972 | finish at 2026-10-17 18:46:37 + [2026-10-15 09:32:02] iteration 1458/ 2384 | consumed samples: 46656 | elapsed time per iteration (ms): 221890.7 | throughput per GPU (TFLOP/s/GPU): 19.1 | MFU 6.11% | FLOPs estimator: workload-aware | logical tokens/s: 75610.3 | valid tokens/s: 75610.3 | physical tokens/s: 76402.3 | loss tokens/s: 59720.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.477200E-01 | lm-global-supervised-tokens: 1.325152E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.271761E-02 | lm-token-buffer-mib: 2.342970E+02 | global_load_balancing_loss: 1.000558E+00 | loss scale: 1.0 | grad norm: 0.102 | params norm: 11170.199 | params rms: 0.060483 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:04:30.744581 | finish at 2026-10-17 18:36:33 + [2026-10-15 09:35:48] iteration 1459/ 2384 | consumed samples: 46688 | elapsed time per iteration (ms): 225273.7 | throughput per GPU (TFLOP/s/GPU): 27.4 | MFU 8.80% | FLOPs estimator: workload-aware | logical tokens/s: 74474.7 | valid tokens/s: 74474.7 | physical tokens/s: 74656.6 | loss tokens/s: 57490.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.169563E-01 | lm-global-supervised-tokens: 1.295098E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.486490E-02 | lm-token-buffer-mib: 2.326639E+02 | global_load_balancing_loss: 1.000614E+00 | loss scale: 1.0 | grad norm: 0.113 | params norm: 11170.182 | params rms: 0.060483 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:52:58.179777 | finish at 2026-10-17 19:28:46 + [2026-10-15 09:39:50] iteration 1460/ 2384 | consumed samples: 46720 | elapsed time per iteration (ms): 241903.5 | throughput per GPU (TFLOP/s/GPU): 19.2 | MFU 6.16% | FLOPs estimator: workload-aware | logical tokens/s: 69355.0 | valid tokens/s: 69355.0 | physical tokens/s: 70060.6 | loss tokens/s: 51168.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.891386E-01 | lm-global-supervised-tokens: 1.237779E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.675444E-02 | lm-token-buffer-mib: 2.335969E+02 | global_load_balancing_loss: 1.000565E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11170.162 | params rms: 0.060483 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:05:18.827685 | finish at 2026-10-17 23:45:08 + [2026-10-15 09:43:32] iteration 1461/ 2384 | consumed samples: 46752 | elapsed time per iteration (ms): 222711.2 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.30% | FLOPs estimator: workload-aware | logical tokens/s: 75331.5 | valid tokens/s: 75331.5 | physical tokens/s: 76550.4 | loss tokens/s: 60342.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.202555E-01 | lm-global-supervised-tokens: 1.343896E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.486220E-02 | lm-token-buffer-mib: 2.339131E+02 | global_load_balancing_loss: 1.000502E+00 | loss scale: 1.0 | grad norm: 0.346 | params norm: 11170.144 | params rms: 0.060483 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:06:02.452343 | finish at 2026-10-17 18:49:35 + [2026-10-15 09:47:14] iteration 1462/ 2384 | consumed samples: 46784 | elapsed time per iteration (ms): 222225.6 | throughput per GPU (TFLOP/s/GPU): 24.1 | MFU 7.73% | FLOPs estimator: workload-aware | logical tokens/s: 75495.9 | valid tokens/s: 75495.9 | physical tokens/s: 75818.9 | loss tokens/s: 54794.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.179566E-01 | lm-global-supervised-tokens: 1.217672E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.597654E-02 | lm-token-buffer-mib: 2.326160E+02 | global_load_balancing_loss: 9.995657E-01 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11170.130 | params rms: 0.060483 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:54:52.035298 | finish at 2026-10-17 18:42:06 + [2026-10-15 09:50:57] iteration 1463/ 2384 | consumed samples: 46816 | elapsed time per iteration (ms): 222092.9 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.11% | FLOPs estimator: workload-aware | logical tokens/s: 75541.4 | valid tokens/s: 75541.4 | physical tokens/s: 76361.8 | loss tokens/s: 55244.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.750063E-01 | lm-global-supervised-tokens: 1.226945E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.119009E-02 | lm-token-buffer-mib: 2.338407E+02 | global_load_balancing_loss: 1.000335E+00 | loss scale: 1.0 | grad norm: 0.852 | params norm: 11170.118 | params rms: 0.060483 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:49:07.565107 | finish at 2026-10-17 18:40:04 + [2026-10-15 09:54:39] iteration 1464/ 2384 | consumed samples: 46848 | elapsed time per iteration (ms): 222238.8 | throughput per GPU (TFLOP/s/GPU): 24.0 | MFU 7.69% | FLOPs estimator: workload-aware | logical tokens/s: 75491.7 | valid tokens/s: 75491.7 | physical tokens/s: 75744.7 | loss tokens/s: 56057.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.690268E-01 | lm-global-supervised-tokens: 1.245818E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.294123E-02 | lm-token-buffer-mib: 2.323789E+02 | global_load_balancing_loss: 1.000842E+00 | loss scale: 1.0 | grad norm: 0.111 | params norm: 11170.103 | params rms: 0.060482 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:47:39.672794 | finish at 2026-10-17 18:42:18 + [2026-10-15 09:58:19] iteration 1465/ 2384 | consumed samples: 46880 | elapsed time per iteration (ms): 220374.2 | throughput per GPU (TFLOP/s/GPU): 19.5 | MFU 6.26% | FLOPs estimator: workload-aware | logical tokens/s: 76130.5 | valid tokens/s: 76130.5 | physical tokens/s: 76762.8 | loss tokens/s: 59046.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.234158E-01 | lm-global-supervised-tokens: 1.301225E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.515376E-02 | lm-token-buffer-mib: 2.346332E+02 | global_load_balancing_loss: 9.996991E-01 | loss scale: 1.0 | grad norm: 0.423 | params norm: 11170.086 | params rms: 0.060482 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:15:23.864919 | finish at 2026-10-17 18:13:43 + [2026-10-15 10:02:04] iteration 1466/ 2384 | consumed samples: 46912 | elapsed time per iteration (ms): 224746.9 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.53% | FLOPs estimator: workload-aware | logical tokens/s: 74649.2 | valid tokens/s: 74649.2 | physical tokens/s: 75496.8 | loss tokens/s: 59105.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.960603E-01 | lm-global-supervised-tokens: 1.328370E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.426167E-02 | lm-token-buffer-mib: 2.359827E+02 | global_load_balancing_loss: 9.998307E-01 | loss scale: 1.0 | grad norm: 0.191 | params norm: 11170.068 | params rms: 0.060482 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:18:37.632388 | finish at 2026-10-17 19:20:42 + [2026-10-15 10:05:50] iteration 1467/ 2384 | consumed samples: 46944 | elapsed time per iteration (ms): 225612.6 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 7.00% | FLOPs estimator: workload-aware | logical tokens/s: 74362.9 | valid tokens/s: 74362.9 | physical tokens/s: 75130.6 | loss tokens/s: 57285.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.338768E-01 | lm-global-supervised-tokens: 1.292424E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.259514E-02 | lm-token-buffer-mib: 2.344998E+02 | global_load_balancing_loss: 1.000575E+00 | loss scale: 1.0 | grad norm: 0.323 | params norm: 11170.048 | params rms: 0.060482 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:28:06.735479 | finish at 2026-10-17 19:33:56 + [2026-10-15 10:09:36] iteration 1468/ 2384 | consumed samples: 46976 | elapsed time per iteration (ms): 226560.1 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.05% | FLOPs estimator: workload-aware | logical tokens/s: 74051.8 | valid tokens/s: 74051.8 | physical tokens/s: 74823.1 | loss tokens/s: 52567.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.885962E-01 | lm-global-supervised-tokens: 1.190973E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.688417E-02 | lm-token-buffer-mib: 2.365027E+02 | global_load_balancing_loss: 1.000112E+00 | loss scale: 1.0 | grad norm: 0.225 | params norm: 11170.024 | params rms: 0.060482 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:38:49.035293 | finish at 2026-10-17 19:48:25 + [2026-10-15 10:13:22] iteration 1469/ 2384 | consumed samples: 47008 | elapsed time per iteration (ms): 225660.6 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.09% | FLOPs estimator: workload-aware | logical tokens/s: 74347.1 | valid tokens/s: 74347.0 | physical tokens/s: 75294.4 | loss tokens/s: 55436.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.048062E-01 | lm-global-supervised-tokens: 1.250983E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 5.064649E-02 | lm-token-buffer-mib: 2.362825E+02 | global_load_balancing_loss: 1.001217E+00 | loss scale: 1.0 | grad norm: 0.174 | params norm: 11169.997 | params rms: 0.060482 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:21:19.488437 | finish at 2026-10-17 19:34:41 + [2026-10-15 10:17:25] iteration 1470/ 2384 | consumed samples: 47040 | elapsed time per iteration (ms): 243364.6 | throughput per GPU (TFLOP/s/GPU): 18.7 | MFU 5.99% | FLOPs estimator: workload-aware | logical tokens/s: 68938.5 | valid tokens/s: 68938.5 | physical tokens/s: 69365.9 | loss tokens/s: 50834.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.454310E-01 | lm-global-supervised-tokens: 1.237125E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.608647E-02 | lm-token-buffer-mib: 2.330529E+02 | global_load_balancing_loss: 1.001039E+00 | loss scale: 1.0 | grad norm: 0.168 | params norm: 11169.969 | params rms: 0.060482 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:47:15.272895 | finish at 2026-10-18 00:04:40 + [2026-10-15 10:21:09] iteration 1471/ 2384 | consumed samples: 47072 | elapsed time per iteration (ms): 223796.4 | throughput per GPU (TFLOP/s/GPU): 23.2 | MFU 7.43% | FLOPs estimator: workload-aware | logical tokens/s: 74966.1 | valid tokens/s: 74966.1 | physical tokens/s: 75563.0 | loss tokens/s: 55633.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.817555E-01 | lm-global-supervised-tokens: 1.245055E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.427831E-02 | lm-token-buffer-mib: 2.352782E+02 | global_load_balancing_loss: 1.000954E+00 | loss scale: 1.0 | grad norm: 0.180 | params norm: 11169.935 | params rms: 0.060482 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:45:26.140474 | finish at 2026-10-17 19:06:35 + [2026-10-15 10:24:50] iteration 1472/ 2384 | consumed samples: 47104 | elapsed time per iteration (ms): 221469.7 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.24% | FLOPs estimator: workload-aware | logical tokens/s: 75753.9 | valid tokens/s: 75753.9 | physical tokens/s: 76253.4 | loss tokens/s: 58895.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.261379E-01 | lm-global-supervised-tokens: 1.304356E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.282472E-02 | lm-token-buffer-mib: 2.331849E+02 | global_load_balancing_loss: 1.000402E+00 | loss scale: 1.0 | grad norm: 0.143 | params norm: 11169.903 | params rms: 0.060481 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:06:20.328873 | finish at 2026-10-17 18:31:11 + [2026-10-15 10:28:29] iteration 1473/ 2384 | consumed samples: 47136 | elapsed time per iteration (ms): 218910.9 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.63% | FLOPs estimator: workload-aware | logical tokens/s: 76639.3 | valid tokens/s: 76639.3 | physical tokens/s: 77297.0 | loss tokens/s: 53964.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.940129E-01 | lm-global-supervised-tokens: 1.181335E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.416064E-02 | lm-token-buffer-mib: 2.330385E+02 | global_load_balancing_loss: 9.997693E-01 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11169.870 | params rms: 0.060481 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 7:23:47.839345 | finish at 2026-10-17 17:52:17 + [2026-10-15 10:32:09] iteration 1474/ 2384 | consumed samples: 47168 | elapsed time per iteration (ms): 219801.6 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.54% | FLOPs estimator: workload-aware | logical tokens/s: 76328.8 | valid tokens/s: 76328.8 | physical tokens/s: 76728.1 | loss tokens/s: 56963.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.780036E-01 | lm-global-supervised-tokens: 1.252064E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.662376E-02 | lm-token-buffer-mib: 2.337114E+02 | global_load_balancing_loss: 1.000087E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11169.833 | params rms: 0.060481 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 7:33:39.456198 | finish at 2026-10-17 18:05:49 + [2026-10-15 10:35:52] iteration 1475/ 2384 | consumed samples: 47200 | elapsed time per iteration (ms): 222634.8 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.97% | FLOPs estimator: workload-aware | logical tokens/s: 75357.4 | valid tokens/s: 75357.4 | physical tokens/s: 76207.6 | loss tokens/s: 60159.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.677834E-01 | lm-global-supervised-tokens: 1.339356E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.835076E-02 | lm-token-buffer-mib: 2.361082E+02 | global_load_balancing_loss: 1.001536E+00 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11169.798 | params rms: 0.060481 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:12:55.030777 | finish at 2026-10-17 18:48:47 + [2026-10-15 10:39:35] iteration 1476/ 2384 | consumed samples: 47232 | elapsed time per iteration (ms): 223010.8 | throughput per GPU (TFLOP/s/GPU): 19.0 | MFU 6.08% | FLOPs estimator: workload-aware | logical tokens/s: 75230.5 | valid tokens/s: 75230.5 | physical tokens/s: 76571.0 | loss tokens/s: 58062.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.456106E-01 | lm-global-supervised-tokens: 1.294849E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.791022E-02 | lm-token-buffer-mib: 2.336841E+02 | global_load_balancing_loss: 9.990932E-01 | loss scale: 1.0 | grad norm: 0.201 | params norm: 11169.763 | params rms: 0.060481 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:14:53.796337 | finish at 2026-10-17 18:54:29 + [2026-10-15 10:43:19] iteration 1477/ 2384 | consumed samples: 47264 | elapsed time per iteration (ms): 223998.0 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.01% | FLOPs estimator: workload-aware | logical tokens/s: 74898.9 | valid tokens/s: 74898.9 | physical tokens/s: 75596.7 | loss tokens/s: 59784.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.237325E-01 | lm-global-supervised-tokens: 1.339168E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.872733E-02 | lm-token-buffer-mib: 2.337637E+02 | global_load_balancing_loss: 1.000180E+00 | loss scale: 1.0 | grad norm: 0.216 | params norm: 11169.726 | params rms: 0.060480 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:26:06.208189 | finish at 2026-10-17 19:09:25 + [2026-10-15 10:46:58] iteration 1478/ 2384 | consumed samples: 47296 | elapsed time per iteration (ms): 219694.7 | throughput per GPU (TFLOP/s/GPU): 19.0 | MFU 6.10% | FLOPs estimator: workload-aware | logical tokens/s: 76366.0 | valid tokens/s: 76365.9 | physical tokens/s: 77018.9 | loss tokens/s: 57925.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.337266E-01 | lm-global-supervised-tokens: 1.272597E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.395744E-02 | lm-token-buffer-mib: 2.331565E+02 | global_load_balancing_loss: 1.000748E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11169.688 | params rms: 0.060480 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 7:17:23.391722 | finish at 2026-10-17 18:04:22 + [2026-10-15 10:50:40] iteration 1479/ 2384 | consumed samples: 47328 | elapsed time per iteration (ms): 221207.1 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.68% | FLOPs estimator: workload-aware | logical tokens/s: 75843.8 | valid tokens/s: 75843.8 | physical tokens/s: 76704.4 | loss tokens/s: 57283.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.003215E-01 | lm-global-supervised-tokens: 1.267154E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.490263E-02 | lm-token-buffer-mib: 2.341455E+02 | global_load_balancing_loss: 1.001047E+00 | loss scale: 1.0 | grad norm: 0.160 | params norm: 11169.649 | params rms: 0.060480 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 7:36:32.433838 | finish at 2026-10-17 18:27:12 + [2026-10-15 10:54:43] iteration 1480/ 2384 | consumed samples: 47360 | elapsed time per iteration (ms): 243409.7 | throughput per GPU (TFLOP/s/GPU): 16.7 | MFU 5.36% | FLOPs estimator: workload-aware | logical tokens/s: 68925.8 | valid tokens/s: 68925.8 | physical tokens/s: 69719.4 | loss tokens/s: 52777.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.653504E-01 | lm-global-supervised-tokens: 1.284648E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.534371E-02 | lm-token-buffer-mib: 2.354587E+02 | global_load_balancing_loss: 9.998085E-01 | loss scale: 1.0 | grad norm: 0.307 | params norm: 11169.607 | params rms: 0.060480 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:07:22.324541 | finish at 2026-10-18 00:02:05 +times across ranks (ms): + save-checkpoint: + rank 0: 23509.03 + rank 1: 23509.01 + rank 2: 23509.03 + rank 3: 23509.07 + rank 4: 23509.00 + rank 5: 23509.05 + rank 6: 23509.01 + rank 7: 23509.01 + rank 8: 23509.90 + rank 9: 23509.93 + rank 10: 23509.91 + rank 11: 23509.89 + rank 12: 23509.95 + rank 13: 23509.90 + rank 14: 23509.90 + rank 15: 23509.90 + rank 16: 23509.32 + rank 17: 23509.30 + rank 18: 23509.30 + rank 19: 23509.28 + rank 20: 23509.32 + rank 21: 23509.30 + rank 22: 23509.33 + rank 23: 23509.32 + rank 24: 23508.75 + rank 25: 23508.81 + rank 26: 23508.78 + rank 27: 23508.79 + rank 28: 23508.73 + rank 29: 23508.77 + rank 30: 23508.73 + rank 31: 23508.75 + rank 32: 23509.14 + rank 33: 23509.18 + rank 34: 23509.12 + rank 35: 23509.16 + rank 36: 23509.11 + rank 37: 23509.15 + rank 38: 23509.11 + rank 39: 23509.11 + rank 40: 23509.34 + rank 41: 23509.37 + rank 42: 23509.35 + rank 43: 23509.40 + rank 44: 23509.35 + rank 45: 23509.33 + rank 46: 23509.33 + rank 47: 23509.33 + rank 48: 23508.91 + rank 49: 23508.93 + rank 50: 23508.93 + rank 51: 23508.91 + rank 52: 23508.92 + rank 53: 23508.93 + rank 54: 23508.90 + rank 55: 23508.91 + rank 56: 23509.03 + rank 57: 23509.02 + rank 58: 23509.02 + rank 59: 23509.03 + rank 60: 23509.06 + rank 61: 23509.03 + rank 62: 23509.01 + rank 63: 23508.99 + [2026-10-15 10:59:02] iteration 1481/ 2384 | consumed samples: 47392 | elapsed time per iteration (ms): 235184.8 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.76% | FLOPs estimator: workload-aware | logical tokens/s: 71336.2 | valid tokens/s: 71336.2 | physical tokens/s: 71971.5 | loss tokens/s: 56090.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.495909E-01 | lm-global-supervised-tokens: 1.319171E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.868185E-01 | lm-token-buffer-mib: 2.343676E+02 | global_load_balancing_loss: 1.000645E+00 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11169.565 | params rms: 0.060480 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:59:31.862477 | finish at 2026-10-17 21:58:34 + [2026-10-15 11:02:50] iteration 1482/ 2384 | consumed samples: 47424 | elapsed time per iteration (ms): 227874.8 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.89% | FLOPs estimator: workload-aware | logical tokens/s: 73624.6 | valid tokens/s: 73624.6 | physical tokens/s: 74126.6 | loss tokens/s: 53439.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.379928E-01 | lm-global-supervised-tokens: 1.217754E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.160890E-02 | lm-token-buffer-mib: 2.344956E+02 | global_load_balancing_loss: 1.000757E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11169.520 | params rms: 0.060479 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:05:43.045054 | finish at 2026-10-17 20:08:33 + [2026-10-15 11:06:39] iteration 1483/ 2384 | consumed samples: 47456 | elapsed time per iteration (ms): 228796.5 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.36% | FLOPs estimator: workload-aware | logical tokens/s: 73327.9 | valid tokens/s: 73327.9 | physical tokens/s: 75107.7 | loss tokens/s: 56324.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.238029E-01 | lm-global-supervised-tokens: 1.288693E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.980743E-02 | lm-token-buffer-mib: 2.421311E+02 | global_load_balancing_loss: 9.999491E-01 | loss scale: 1.0 | grad norm: 0.394 | params norm: 11169.474 | params rms: 0.060479 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:15:45.641315 | finish at 2026-10-17 20:22:24 + [2026-10-15 11:10:18] iteration 1484/ 2384 | consumed samples: 47488 | elapsed time per iteration (ms): 219884.4 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.29% | FLOPs estimator: workload-aware | logical tokens/s: 76300.0 | valid tokens/s: 76300.0 | physical tokens/s: 76579.6 | loss tokens/s: 56356.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.844830E-01 | lm-global-supervised-tokens: 1.239197E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.838183E-02 | lm-token-buffer-mib: 2.324361E+02 | global_load_balancing_loss: 9.998768E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11169.429 | params rms: 0.060479 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:58:15.966768 | finish at 2026-10-17 18:08:34 + [2026-10-15 11:14:05] iteration 1485/ 2384 | consumed samples: 47520 | elapsed time per iteration (ms): 226653.4 | throughput per GPU (TFLOP/s/GPU): 23.9 | MFU 7.66% | FLOPs estimator: workload-aware | logical tokens/s: 74021.3 | valid tokens/s: 74021.3 | physical tokens/s: 74786.4 | loss tokens/s: 58388.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.968041E-01 | lm-global-supervised-tokens: 1.323404E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.234461E-02 | lm-token-buffer-mib: 2.341949E+02 | global_load_balancing_loss: 1.001330E+00 | loss scale: 1.0 | grad norm: 0.160 | params norm: 11169.383 | params rms: 0.060479 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:36:01.438701 | finish at 2026-10-17 19:50:07 + [2026-10-15 11:17:45] iteration 1486/ 2384 | consumed samples: 47552 | elapsed time per iteration (ms): 220148.2 | throughput per GPU (TFLOP/s/GPU): 25.0 | MFU 8.02% | FLOPs estimator: workload-aware | logical tokens/s: 76208.5 | valid tokens/s: 76208.5 | physical tokens/s: 76436.4 | loss tokens/s: 56304.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.770707E-01 | lm-global-supervised-tokens: 1.239535E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.474000E-02 | lm-token-buffer-mib: 2.331679E+02 | global_load_balancing_loss: 9.999237E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11169.336 | params rms: 0.060478 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:54:53.056441 | finish at 2026-10-17 18:12:38 + [2026-10-15 11:21:27] iteration 1487/ 2384 | consumed samples: 47584 | elapsed time per iteration (ms): 221480.7 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.03% | FLOPs estimator: workload-aware | logical tokens/s: 75750.0 | valid tokens/s: 75750.0 | physical tokens/s: 76509.3 | loss tokens/s: 57893.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.152225E-01 | lm-global-supervised-tokens: 1.282233E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.201789E-02 | lm-token-buffer-mib: 2.334568E+02 | global_load_balancing_loss: 1.000386E+00 | loss scale: 1.0 | grad norm: 0.146 | params norm: 11169.288 | params rms: 0.060478 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 7:11:08.224699 | finish at 2026-10-17 18:32:35 + [2026-10-15 11:25:12] iteration 1488/ 2384 | consumed samples: 47616 | elapsed time per iteration (ms): 225391.3 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.71% | FLOPs estimator: workload-aware | logical tokens/s: 74435.8 | valid tokens/s: 74435.8 | physical tokens/s: 75047.0 | loss tokens/s: 56059.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.789199E-01 | lm-global-supervised-tokens: 1.263542E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 6.431577E-02 | lm-token-buffer-mib: 2.346189E+02 | global_load_balancing_loss: 1.000172E+00 | loss scale: 1.0 | grad norm: 0.165 | params norm: 11169.242 | params rms: 0.060478 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:05:50.624207 | finish at 2026-10-17 19:31:03 + [2026-10-15 11:28:54] iteration 1489/ 2384 | consumed samples: 47648 | elapsed time per iteration (ms): 221591.4 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.75% | FLOPs estimator: workload-aware | logical tokens/s: 75712.2 | valid tokens/s: 75712.2 | physical tokens/s: 76393.2 | loss tokens/s: 51966.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.779920E-01 | lm-global-supervised-tokens: 1.151541E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.911368E-02 | lm-token-buffer-mib: 2.333787E+02 | global_load_balancing_loss: 1.000447E+00 | loss scale: 1.0 | grad norm: 0.183 | params norm: 11169.197 | params rms: 0.060478 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 7:05:24.280086 | finish at 2026-10-17 18:34:18 + [2026-10-15 11:32:57] iteration 1490/ 2384 | consumed samples: 47680 | elapsed time per iteration (ms): 243565.9 | throughput per GPU (TFLOP/s/GPU): 18.1 | MFU 5.80% | FLOPs estimator: workload-aware | logical tokens/s: 68881.6 | valid tokens/s: 68881.6 | physical tokens/s: 69350.4 | loss tokens/s: 48965.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.300278E-01 | lm-global-supervised-tokens: 1.192626E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.369513E-02 | lm-token-buffer-mib: 2.347561E+02 | global_load_balancing_loss: 9.990367E-01 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11169.147 | params rms: 0.060477 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:29:07.917236 | finish at 2026-10-18 00:02:05 + [2026-10-15 11:36:40] iteration 1491/ 2384 | consumed samples: 47712 | elapsed time per iteration (ms): 222322.9 | throughput per GPU (TFLOP/s/GPU): 24.2 | MFU 7.76% | FLOPs estimator: workload-aware | logical tokens/s: 75463.2 | valid tokens/s: 75463.2 | physical tokens/s: 76223.5 | loss tokens/s: 59407.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.469973E-01 | lm-global-supervised-tokens: 1.320763E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.290235E-02 | lm-token-buffer-mib: 2.327942E+02 | global_load_balancing_loss: 9.999794E-01 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11169.096 | params rms: 0.060477 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 7:08:54.392971 | finish at 2026-10-17 18:45:34 + [2026-10-15 11:40:26] iteration 1492/ 2384 | consumed samples: 47744 | elapsed time per iteration (ms): 226445.0 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.98% | FLOPs estimator: workload-aware | logical tokens/s: 74089.5 | valid tokens/s: 74089.5 | physical tokens/s: 75508.1 | loss tokens/s: 53744.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.542030E-01 | lm-global-supervised-tokens: 1.217027E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.924137E-02 | lm-token-buffer-mib: 2.371265E+02 | global_load_balancing_loss: 1.000417E+00 | loss scale: 1.0 | grad norm: 0.146 | params norm: 11169.046 | params rms: 0.060477 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:06:28.977158 | finish at 2026-10-17 19:46:55 + [2026-10-15 11:44:13] iteration 1493/ 2384 | consumed samples: 47776 | elapsed time per iteration (ms): 226557.0 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.24% | FLOPs estimator: workload-aware | logical tokens/s: 74052.9 | valid tokens/s: 74052.9 | physical tokens/s: 74567.1 | loss tokens/s: 58261.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.170213E-01 | lm-global-supervised-tokens: 1.319961E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.151098E-02 | lm-token-buffer-mib: 2.338696E+02 | global_load_balancing_loss: 1.000917E+00 | loss scale: 1.0 | grad norm: 0.227 | params norm: 11168.994 | params rms: 0.060476 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:04:22.300313 | finish at 2026-10-17 19:48:35 + [2026-10-15 11:47:54] iteration 1494/ 2384 | consumed samples: 47808 | elapsed time per iteration (ms): 221568.8 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.18% | FLOPs estimator: workload-aware | logical tokens/s: 75719.8 | valid tokens/s: 75719.8 | physical tokens/s: 76628.3 | loss tokens/s: 55371.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.883456E-01 | lm-global-supervised-tokens: 1.226863E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.111292E-02 | lm-token-buffer-mib: 2.338828E+02 | global_load_balancing_loss: 1.001576E+00 | loss scale: 1.0 | grad norm: 0.195 | params norm: 11168.944 | params rms: 0.060476 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:46:36.214364 | finish at 2026-10-17 18:34:30 + [2026-10-15 11:51:43] iteration 1495/ 2384 | consumed samples: 47840 | elapsed time per iteration (ms): 229145.0 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.74% | FLOPs estimator: workload-aware | logical tokens/s: 73216.5 | valid tokens/s: 73216.5 | physical tokens/s: 74240.8 | loss tokens/s: 55795.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.012343E-01 | lm-global-supervised-tokens: 1.278522E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.629321E-02 | lm-token-buffer-mib: 2.366231E+02 | global_load_balancing_loss: 1.000503E+00 | loss scale: 1.0 | grad norm: 0.129 | params norm: 11168.898 | params rms: 0.060476 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:35:09.866619 | finish at 2026-10-17 20:26:53 + [2026-10-15 11:55:25] iteration 1496/ 2384 | consumed samples: 47872 | elapsed time per iteration (ms): 221846.7 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.05% | FLOPs estimator: workload-aware | logical tokens/s: 75625.3 | valid tokens/s: 75625.3 | physical tokens/s: 76027.1 | loss tokens/s: 57309.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.073849E-01 | lm-global-supervised-tokens: 1.271392E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.380838E-02 | lm-token-buffer-mib: 2.336673E+02 | global_load_balancing_loss: 1.000404E+00 | loss scale: 1.0 | grad norm: 0.239 | params norm: 11168.852 | params rms: 0.060476 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:43:19.848810 | finish at 2026-10-17 18:38:45 + [2026-10-15 11:59:12] iteration 1497/ 2384 | consumed samples: 47904 | elapsed time per iteration (ms): 227008.1 | throughput per GPU (TFLOP/s/GPU): 23.2 | MFU 7.43% | FLOPs estimator: workload-aware | logical tokens/s: 73905.8 | valid tokens/s: 73905.8 | physical tokens/s: 75080.9 | loss tokens/s: 52329.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.148738E-01 | lm-global-supervised-tokens: 1.187918E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.637309E-02 | lm-token-buffer-mib: 2.389852E+02 | global_load_balancing_loss: 9.999322E-01 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11168.809 | params rms: 0.060475 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 7:55:56.143491 | finish at 2026-10-17 19:55:08 + [2026-10-15 12:02:56] iteration 1498/ 2384 | consumed samples: 47936 | elapsed time per iteration (ms): 223691.9 | throughput per GPU (TFLOP/s/GPU): 18.6 | MFU 5.96% | FLOPs estimator: workload-aware | logical tokens/s: 75001.4 | valid tokens/s: 75001.4 | physical tokens/s: 76488.6 | loss tokens/s: 55893.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.452466E-01 | lm-global-supervised-tokens: 1.250293E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.026045E-02 | lm-token-buffer-mib: 2.389145E+02 | global_load_balancing_loss: 9.999400E-01 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11168.766 | params rms: 0.060475 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 7:03:11.026582 | finish at 2026-10-17 19:06:07 + [2026-10-15 12:06:37] iteration 1499/ 2384 | consumed samples: 47968 | elapsed time per iteration (ms): 221169.6 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.53% | FLOPs estimator: workload-aware | logical tokens/s: 75856.5 | valid tokens/s: 75856.5 | physical tokens/s: 76475.5 | loss tokens/s: 56151.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.730292E-01 | lm-global-supervised-tokens: 1.241901E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.539236E-02 | lm-token-buffer-mib: 2.349169E+02 | global_load_balancing_loss: 9.995831E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11168.721 | params rms: 0.060475 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:22:15.133356 | finish at 2026-10-17 18:28:52 + [2026-10-15 12:10:43] iteration 1500/ 2384 | consumed samples: 48000 | elapsed time per iteration (ms): 246301.6 | throughput per GPU (TFLOP/s/GPU): 23.9 | MFU 7.67% | FLOPs estimator: workload-aware | logical tokens/s: 68116.5 | valid tokens/s: 68116.5 | physical tokens/s: 68670.5 | loss tokens/s: 52607.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.217322E-01 | lm-global-supervised-tokens: 1.295739E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 6.142940E-02 | lm-token-buffer-mib: 2.324032E+02 | global_load_balancing_loss: 9.996913E-01 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11168.673 | params rms: 0.060475 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:28:50.596889 | finish at 2026-10-18 00:39:34 + [2026-10-15 12:14:25] iteration 1501/ 2384 | consumed samples: 48032 | elapsed time per iteration (ms): 221954.5 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.81% | FLOPs estimator: workload-aware | logical tokens/s: 75588.5 | valid tokens/s: 75588.5 | physical tokens/s: 76480.1 | loss tokens/s: 57010.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.513578E-01 | lm-global-supervised-tokens: 1.265364E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.216917E-02 | lm-token-buffer-mib: 2.345889E+02 | global_load_balancing_loss: 1.001475E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11168.623 | params rms: 0.060474 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:26:25.782413 | finish at 2026-10-17 18:40:51 + [2026-10-15 12:18:06] iteration 1502/ 2384 | consumed samples: 48064 | elapsed time per iteration (ms): 220374.3 | throughput per GPU (TFLOP/s/GPU): 19.2 | MFU 6.16% | FLOPs estimator: workload-aware | logical tokens/s: 76130.5 | valid tokens/s: 76130.5 | physical tokens/s: 76988.1 | loss tokens/s: 56008.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.336543E-01 | lm-global-supervised-tokens: 1.234293E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.290879E-02 | lm-token-buffer-mib: 2.328305E+02 | global_load_balancing_loss: 1.000770E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11168.575 | params rms: 0.060474 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:59:30.132182 | finish at 2026-10-17 18:17:36 + [2026-10-15 12:21:50] iteration 1503/ 2384 | consumed samples: 48096 | elapsed time per iteration (ms): 224049.7 | throughput per GPU (TFLOP/s/GPU): 24.3 | MFU 7.80% | FLOPs estimator: workload-aware | logical tokens/s: 74881.4 | valid tokens/s: 74881.4 | physical tokens/s: 75568.7 | loss tokens/s: 58711.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.665882E-01 | lm-global-supervised-tokens: 1.315422E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.745053E-02 | lm-token-buffer-mib: 2.332622E+02 | global_load_balancing_loss: 1.000441E+00 | loss scale: 1.0 | grad norm: 0.192 | params norm: 11168.530 | params rms: 0.060474 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:49:47.785089 | finish at 2026-10-17 19:11:38 + [2026-10-15 12:25:30] iteration 1504/ 2384 | consumed samples: 48128 | elapsed time per iteration (ms): 220556.4 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.12% | FLOPs estimator: workload-aware | logical tokens/s: 76067.6 | valid tokens/s: 76067.6 | physical tokens/s: 76477.7 | loss tokens/s: 58097.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.667706E-01 | lm-global-supervised-tokens: 1.281375E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.624940E-02 | lm-token-buffer-mib: 2.328988E+02 | global_load_balancing_loss: 1.001419E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11168.483 | params rms: 0.060474 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:54:49.601212 | finish at 2026-10-17 18:20:20 + [2026-10-15 12:29:12] iteration 1505/ 2384 | consumed samples: 48160 | elapsed time per iteration (ms): 221824.6 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.35% | FLOPs estimator: workload-aware | logical tokens/s: 75632.8 | valid tokens/s: 75632.8 | physical tokens/s: 76007.0 | loss tokens/s: 57522.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.784745E-01 | lm-global-supervised-tokens: 1.275984E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.519191E-02 | lm-token-buffer-mib: 2.330872E+02 | global_load_balancing_loss: 1.001779E+00 | loss scale: 1.0 | grad norm: 0.160 | params norm: 11168.441 | params rms: 0.060473 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:09:43.866974 | finish at 2026-10-17 18:38:56 + [2026-10-15 12:32:56] iteration 1506/ 2384 | consumed samples: 48192 | elapsed time per iteration (ms): 223808.7 | throughput per GPU (TFLOP/s/GPU): 23.7 | MFU 7.61% | FLOPs estimator: workload-aware | logical tokens/s: 74962.2 | valid tokens/s: 74962.2 | physical tokens/s: 75345.8 | loss tokens/s: 54149.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.176627E-01 | lm-global-supervised-tokens: 1.211922E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.489012E-02 | lm-token-buffer-mib: 2.334510E+02 | global_load_balancing_loss: 9.997964E-01 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11168.400 | params rms: 0.060473 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:35:04.025905 | finish at 2026-10-17 19:08:00 + [2026-10-15 12:36:40] iteration 1507/ 2384 | consumed samples: 48224 | elapsed time per iteration (ms): 223582.5 | throughput per GPU (TFLOP/s/GPU): 24.1 | MFU 7.72% | FLOPs estimator: workload-aware | logical tokens/s: 75038.0 | valid tokens/s: 75038.0 | physical tokens/s: 75468.1 | loss tokens/s: 57772.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.880169E-01 | lm-global-supervised-tokens: 1.291695E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.501165E-02 | lm-token-buffer-mib: 2.330342E+02 | global_load_balancing_loss: 9.986320E-01 | loss scale: 1.0 | grad norm: 0.163 | params norm: 11168.359 | params rms: 0.060473 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:28:01.831365 | finish at 2026-10-17 19:04:41 + [2026-10-15 12:40:23] iteration 1508/ 2384 | consumed samples: 48256 | elapsed time per iteration (ms): 223065.8 | throughput per GPU (TFLOP/s/GPU): 24.9 | MFU 7.97% | FLOPs estimator: workload-aware | logical tokens/s: 75211.9 | valid tokens/s: 75211.9 | physical tokens/s: 75652.6 | loss tokens/s: 55564.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.936148E-01 | lm-global-supervised-tokens: 1.239462E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.263038E-02 | lm-token-buffer-mib: 2.339139E+02 | global_load_balancing_loss: 9.978765E-01 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11168.316 | params rms: 0.060473 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:16:45.654542 | finish at 2026-10-17 18:57:08 + [2026-10-15 12:44:06] iteration 1509/ 2384 | consumed samples: 48288 | elapsed time per iteration (ms): 223761.8 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.70% | FLOPs estimator: workload-aware | logical tokens/s: 74977.9 | valid tokens/s: 74977.9 | physical tokens/s: 75699.9 | loss tokens/s: 57724.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.172156E-01 | lm-global-supervised-tokens: 1.291662E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.559815E-02 | lm-token-buffer-mib: 2.351353E+02 | global_load_balancing_loss: 9.996821E-01 | loss scale: 1.0 | grad norm: 0.174 | params norm: 11168.273 | params rms: 0.060473 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:23:11.576922 | finish at 2026-10-17 19:07:18 + [2026-10-15 12:48:14] iteration 1510/ 2384 | consumed samples: 48320 | elapsed time per iteration (ms): 247544.4 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.75% | FLOPs estimator: workload-aware | logical tokens/s: 67774.5 | valid tokens/s: 67774.5 | physical tokens/s: 68765.8 | loss tokens/s: 50431.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.994563E-01 | lm-global-supervised-tokens: 1.248393E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.429899E-02 | lm-token-buffer-mib: 2.351345E+02 | global_load_balancing_loss: 1.001064E+00 | loss scale: 1.0 | grad norm: 0.257 | params norm: 11168.234 | params rms: 0.060472 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:05:53.787868 | finish at 2026-10-18 00:54:08 + [2026-10-15 12:51:56] iteration 1511/ 2384 | consumed samples: 48352 | elapsed time per iteration (ms): 222583.7 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.21% | FLOPs estimator: workload-aware | logical tokens/s: 75374.7 | valid tokens/s: 75374.7 | physical tokens/s: 75907.6 | loss tokens/s: 60177.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.915797E-01 | lm-global-supervised-tokens: 1.339463E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.299297E-02 | lm-token-buffer-mib: 2.334943E+02 | global_load_balancing_loss: 1.001622E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11168.196 | params rms: 0.060472 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:58:35.603143 | finish at 2026-10-17 18:50:32 + [2026-10-15 12:55:35] iteration 1512/ 2384 | consumed samples: 48384 | elapsed time per iteration (ms): 218608.6 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.90% | FLOPs estimator: workload-aware | logical tokens/s: 76745.4 | valid tokens/s: 76745.4 | physical tokens/s: 77845.4 | loss tokens/s: 61221.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.909652E-01 | lm-global-supervised-tokens: 1.338350E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.449134E-02 | lm-token-buffer-mib: 2.335104E+02 | global_load_balancing_loss: 1.001246E+00 | loss scale: 1.0 | grad norm: 0.147 | params norm: 11168.159 | params rms: 0.060472 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:57:06.661900 | finish at 2026-10-17 17:52:42 + [2026-10-15 12:59:16] iteration 1513/ 2384 | consumed samples: 48416 | elapsed time per iteration (ms): 220976.8 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.34% | FLOPs estimator: workload-aware | logical tokens/s: 75922.9 | valid tokens/s: 75922.9 | physical tokens/s: 76844.5 | loss tokens/s: 58234.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.587219E-01 | lm-global-supervised-tokens: 1.286856E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.423687E-02 | lm-token-buffer-mib: 2.330499E+02 | global_load_balancing_loss: 1.001608E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11168.122 | params rms: 0.060472 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:27:50.819558 | finish at 2026-10-17 18:27:07 + [2026-10-15 13:02:57] iteration 1514/ 2384 | consumed samples: 48448 | elapsed time per iteration (ms): 220891.5 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.30% | FLOPs estimator: workload-aware | logical tokens/s: 75952.1 | valid tokens/s: 75952.1 | physical tokens/s: 76407.2 | loss tokens/s: 53610.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.205671E-01 | lm-global-supervised-tokens: 1.184217E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.439133E-02 | lm-token-buffer-mib: 2.333573E+02 | global_load_balancing_loss: 1.001346E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11168.083 | params rms: 0.060472 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:22:55.632584 | finish at 2026-10-17 18:25:53 + [2026-10-15 13:06:40] iteration 1515/ 2384 | consumed samples: 48480 | elapsed time per iteration (ms): 222660.1 | throughput per GPU (TFLOP/s/GPU): 24.7 | MFU 7.93% | FLOPs estimator: workload-aware | logical tokens/s: 75348.9 | valid tokens/s: 75348.9 | physical tokens/s: 76363.1 | loss tokens/s: 53875.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.474756E-01 | lm-global-supervised-tokens: 1.199584E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.057731E-02 | lm-token-buffer-mib: 2.334580E+02 | global_load_balancing_loss: 1.002188E+00 | loss scale: 1.0 | grad norm: 0.114 | params norm: 11168.042 | params rms: 0.060471 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:44:51.633723 | finish at 2026-10-17 18:51:31 + [2026-10-15 13:10:24] iteration 1516/ 2384 | consumed samples: 48512 | elapsed time per iteration (ms): 223931.1 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.84% | FLOPs estimator: workload-aware | logical tokens/s: 74921.1 | valid tokens/s: 74921.1 | physical tokens/s: 75851.0 | loss tokens/s: 58767.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.677670E-01 | lm-global-supervised-tokens: 1.315994E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.470214E-02 | lm-token-buffer-mib: 2.359654E+02 | global_load_balancing_loss: 9.981815E-01 | loss scale: 1.0 | grad norm: 0.143 | params norm: 11168.001 | params rms: 0.060471 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:59:32.184359 | finish at 2026-10-17 19:09:56 + [2026-10-15 13:14:05] iteration 1517/ 2384 | consumed samples: 48544 | elapsed time per iteration (ms): 221011.5 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.30% | FLOPs estimator: workload-aware | logical tokens/s: 75910.9 | valid tokens/s: 75910.9 | physical tokens/s: 76432.6 | loss tokens/s: 57011.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.969502E-01 | lm-global-supervised-tokens: 1.260020E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.656759E-02 | lm-token-buffer-mib: 2.344779E+02 | global_load_balancing_loss: 9.981499E-01 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11167.963 | params rms: 0.060471 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:13:36.975358 | finish at 2026-10-17 18:27:42 + [2026-10-15 13:17:50] iteration 1518/ 2384 | consumed samples: 48576 | elapsed time per iteration (ms): 225063.9 | throughput per GPU (TFLOP/s/GPU): 23.2 | MFU 7.45% | FLOPs estimator: workload-aware | logical tokens/s: 74544.2 | valid tokens/s: 74544.2 | physical tokens/s: 75882.2 | loss tokens/s: 57522.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.274447E-01 | lm-global-supervised-tokens: 1.294612E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.087097E-02 | lm-token-buffer-mib: 2.355746E+02 | global_load_balancing_loss: 9.999256E-01 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11167.924 | params rms: 0.060471 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:08:25.336982 | finish at 2026-10-17 19:26:15 + [2026-10-15 13:21:34] iteration 1519/ 2384 | consumed samples: 48608 | elapsed time per iteration (ms): 224026.1 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.63% | FLOPs estimator: workload-aware | logical tokens/s: 74889.5 | valid tokens/s: 74889.5 | physical tokens/s: 75625.2 | loss tokens/s: 52444.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.499135E-01 | lm-global-supervised-tokens: 1.174890E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.286729E-02 | lm-token-buffer-mib: 2.357897E+02 | global_load_balancing_loss: 1.000514E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11167.886 | params rms: 0.060470 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:49:42.575194 | finish at 2026-10-17 19:11:16 + [2026-10-15 13:25:37] iteration 1520/ 2384 | consumed samples: 48640 | elapsed time per iteration (ms): 243818.9 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.81% | FLOPs estimator: workload-aware | logical tokens/s: 68810.0 | valid tokens/s: 68810.0 | physical tokens/s: 69222.2 | loss tokens/s: 50036.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.857533E-01 | lm-global-supervised-tokens: 1.219975E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.597598E-02 | lm-token-buffer-mib: 2.330747E+02 | global_load_balancing_loss: 1.000217E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11167.845 | params rms: 0.060470 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:30:59.569450 | finish at 2026-10-17 23:56:37 +times across ranks (ms): + save-checkpoint: + rank 0: 24981.93 + rank 1: 24981.99 + rank 2: 24981.98 + rank 3: 24981.97 + rank 4: 24981.91 + rank 5: 24981.95 + rank 6: 24981.92 + rank 7: 24981.91 + rank 8: 24981.91 + rank 9: 24981.92 + rank 10: 24981.92 + rank 11: 24981.91 + rank 12: 24981.88 + rank 13: 24981.92 + rank 14: 24981.89 + rank 15: 24981.89 + rank 16: 24981.91 + rank 17: 24981.88 + rank 18: 24981.89 + rank 19: 24981.92 + rank 20: 24981.85 + rank 21: 24981.86 + rank 22: 24981.88 + rank 23: 24981.89 + rank 24: 24981.96 + rank 25: 24981.95 + rank 26: 24981.98 + rank 27: 24981.97 + rank 28: 24981.83 + rank 29: 24981.84 + rank 30: 24981.98 + rank 31: 24981.93 + rank 32: 24980.27 + rank 33: 24980.28 + rank 34: 24980.25 + rank 35: 24980.27 + rank 36: 24980.23 + rank 37: 24980.26 + rank 38: 24980.25 + rank 39: 24980.27 + rank 40: 24983.52 + rank 41: 24983.49 + rank 42: 24983.49 + rank 43: 24983.51 + rank 44: 24983.62 + rank 45: 24983.48 + rank 46: 24983.50 + rank 47: 24983.50 + rank 48: 24982.54 + rank 49: 24982.53 + rank 50: 24982.53 + rank 51: 24982.59 + rank 52: 24982.47 + rank 53: 24982.50 + rank 54: 24982.47 + rank 55: 24982.52 + rank 56: 24981.93 + rank 57: 24981.93 + rank 58: 24981.95 + rank 59: 24981.92 + rank 60: 24981.96 + rank 61: 24981.89 + rank 62: 24981.90 + rank 63: 24981.90 + [2026-10-15 13:29:44] iteration 1521/ 2384 | consumed samples: 48672 | elapsed time per iteration (ms): 221940.0 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.10% | FLOPs estimator: workload-aware | logical tokens/s: 75593.4 | valid tokens/s: 75593.4 | physical tokens/s: 76332.3 | loss tokens/s: 57645.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.485318E-01 | lm-global-supervised-tokens: 1.279379E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.394714E-02 | lm-token-buffer-mib: 2.339561E+02 | global_load_balancing_loss: 9.994969E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11167.801 | params rms: 0.060470 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:12:14.211613 | finish at 2026-10-17 18:41:59 + [2026-10-15 13:33:32] iteration 1522/ 2384 | consumed samples: 48704 | elapsed time per iteration (ms): 227936.3 | throughput per GPU (TFLOP/s/GPU): 18.4 | MFU 5.90% | FLOPs estimator: workload-aware | logical tokens/s: 73604.8 | valid tokens/s: 73604.8 | physical tokens/s: 74400.9 | loss tokens/s: 57271.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.536387E-01 | lm-global-supervised-tokens: 1.305417E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.316000E-02 | lm-token-buffer-mib: 2.354730E+02 | global_load_balancing_loss: 1.000396E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11167.761 | params rms: 0.060470 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:34:41.097416 | finish at 2026-10-17 20:08:13 + [2026-10-15 13:37:14] iteration 1523/ 2384 | consumed samples: 48736 | elapsed time per iteration (ms): 221128.0 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.31% | FLOPs estimator: workload-aware | logical tokens/s: 75870.8 | valid tokens/s: 75870.8 | physical tokens/s: 76288.4 | loss tokens/s: 53951.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.928665E-01 | lm-global-supervised-tokens: 1.193011E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.284360E-02 | lm-token-buffer-mib: 2.326584E+02 | global_load_balancing_loss: 1.001567E+00 | loss scale: 1.0 | grad norm: 0.127 | params norm: 11167.722 | params rms: 0.060470 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:53:11.173942 | finish at 2026-10-17 18:30:25 + [2026-10-15 13:40:56] iteration 1524/ 2384 | consumed samples: 48768 | elapsed time per iteration (ms): 222754.2 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.63% | FLOPs estimator: workload-aware | logical tokens/s: 75317.1 | valid tokens/s: 75317.1 | physical tokens/s: 76373.1 | loss tokens/s: 56360.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.462935E-01 | lm-global-supervised-tokens: 1.255448E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.334862E-02 | lm-token-buffer-mib: 2.369706E+02 | global_load_balancing_loss: 1.001440E+00 | loss scale: 1.0 | grad norm: 0.349 | params norm: 11167.680 | params rms: 0.060469 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:12:48.586969 | finish at 2026-10-17 18:53:45 + [2026-10-15 13:44:35] iteration 1525/ 2384 | consumed samples: 48800 | elapsed time per iteration (ms): 218408.9 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.66% | FLOPs estimator: workload-aware | logical tokens/s: 76815.6 | valid tokens/s: 76815.6 | physical tokens/s: 77176.0 | loss tokens/s: 61349.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.122403E-01 | lm-global-supervised-tokens: 1.339921E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.969311E-02 | lm-token-buffer-mib: 2.329475E+02 | global_load_balancing_loss: 1.001016E+00 | loss scale: 1.0 | grad norm: 0.171 | params norm: 11167.638 | params rms: 0.060469 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:06:53.267852 | finish at 2026-10-17 17:51:28 + [2026-10-15 13:48:17] iteration 1526/ 2384 | consumed samples: 48832 | elapsed time per iteration (ms): 222228.5 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.15% | FLOPs estimator: workload-aware | logical tokens/s: 75495.2 | valid tokens/s: 75495.2 | physical tokens/s: 75999.6 | loss tokens/s: 57820.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.018999E-01 | lm-global-supervised-tokens: 1.284932E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.065392E-02 | lm-token-buffer-mib: 2.340852E+02 | global_load_balancing_loss: 9.993243E-01 | loss scale: 1.0 | grad norm: 0.222 | params norm: 11167.599 | params rms: 0.060469 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:57:52.023653 | finish at 2026-10-17 18:46:09 + [2026-10-15 13:51:59] iteration 1527/ 2384 | consumed samples: 48864 | elapsed time per iteration (ms): 221836.5 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.38% | FLOPs estimator: workload-aware | logical tokens/s: 75628.6 | valid tokens/s: 75628.6 | physical tokens/s: 76719.0 | loss tokens/s: 59595.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.013068E-01 | lm-global-supervised-tokens: 1.322052E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.424866E-02 | lm-token-buffer-mib: 2.350081E+02 | global_load_balancing_loss: 9.990787E-01 | loss scale: 1.0 | grad norm: 0.483 | params norm: 11167.557 | params rms: 0.060469 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:48:33.856125 | finish at 2026-10-17 18:40:33 + [2026-10-15 13:55:41] iteration 1528/ 2384 | consumed samples: 48896 | elapsed time per iteration (ms): 222665.2 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.84% | FLOPs estimator: workload-aware | logical tokens/s: 75347.1 | valid tokens/s: 75347.1 | physical tokens/s: 75749.4 | loss tokens/s: 55258.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.178122E-01 | lm-global-supervised-tokens: 1.230417E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.592743E-02 | lm-token-buffer-mib: 2.338348E+02 | global_load_balancing_loss: 9.994106E-01 | loss scale: 1.0 | grad norm: 0.217 | params norm: 11167.515 | params rms: 0.060468 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:56:41.387930 | finish at 2026-10-17 18:52:23 + [2026-10-15 13:59:26] iteration 1529/ 2384 | consumed samples: 48928 | elapsed time per iteration (ms): 224168.6 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.93% | FLOPs estimator: workload-aware | logical tokens/s: 74841.8 | valid tokens/s: 74841.8 | physical tokens/s: 75541.2 | loss tokens/s: 56460.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.969153E-01 | lm-global-supervised-tokens: 1.265660E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.174188E-02 | lm-token-buffer-mib: 2.335046E+02 | global_load_balancing_loss: 1.001307E+00 | loss scale: 1.0 | grad norm: 0.189 | params norm: 11167.478 | params rms: 0.060468 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:14:24.130647 | finish at 2026-10-17 19:13:50 + [2026-10-15 14:03:27] iteration 1530/ 2384 | consumed samples: 48960 | elapsed time per iteration (ms): 241430.7 | throughput per GPU (TFLOP/s/GPU): 16.8 | MFU 5.37% | FLOPs estimator: workload-aware | logical tokens/s: 69490.8 | valid tokens/s: 69490.8 | physical tokens/s: 70488.1 | loss tokens/s: 53731.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.882491E-01 | lm-global-supervised-tokens: 1.297244E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.344407E-02 | lm-token-buffer-mib: 2.334303E+02 | global_load_balancing_loss: 1.000739E+00 | loss scale: 1.0 | grad norm: 0.222 | params norm: 11167.438 | params rms: 0.060468 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:16:21.797493 | finish at 2026-10-17 23:19:49 + [2026-10-15 14:07:17] iteration 1531/ 2384 | consumed samples: 48992 | elapsed time per iteration (ms): 229654.1 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.26% | FLOPs estimator: workload-aware | logical tokens/s: 73054.2 | valid tokens/s: 73054.2 | physical tokens/s: 73611.4 | loss tokens/s: 57744.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.043787E-01 | lm-global-supervised-tokens: 1.326114E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.777834E-02 | lm-token-buffer-mib: 2.340415E+02 | global_load_balancing_loss: 1.000651E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11167.401 | params rms: 0.060468 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:24:54.987111 | finish at 2026-10-17 20:32:12 + [2026-10-15 14:10:59] iteration 1532/ 2384 | consumed samples: 49024 | elapsed time per iteration (ms): 221898.2 | throughput per GPU (TFLOP/s/GPU): 25.0 | MFU 8.02% | FLOPs estimator: workload-aware | logical tokens/s: 75607.6 | valid tokens/s: 75607.6 | physical tokens/s: 76302.8 | loss tokens/s: 58055.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.465775E-01 | lm-global-supervised-tokens: 1.288232E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.280433E-02 | lm-token-buffer-mib: 2.333306E+02 | global_load_balancing_loss: 1.002579E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11167.363 | params rms: 0.060468 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:30:57.277805 | finish at 2026-10-17 18:41:56 + [2026-10-15 14:14:42] iteration 1533/ 2384 | consumed samples: 49056 | elapsed time per iteration (ms): 223920.8 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.99% | FLOPs estimator: workload-aware | logical tokens/s: 74924.6 | valid tokens/s: 74924.6 | physical tokens/s: 75460.7 | loss tokens/s: 57009.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.197387E-01 | lm-global-supervised-tokens: 1.276555E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.265209E-02 | lm-token-buffer-mib: 2.345513E+02 | global_load_balancing_loss: 1.000259E+00 | loss scale: 1.0 | grad norm: 0.159 | params norm: 11167.324 | params rms: 0.060467 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:55:56.621673 | finish at 2026-10-17 19:10:39 + [2026-10-15 14:18:29] iteration 1534/ 2384 | consumed samples: 49088 | elapsed time per iteration (ms): 226180.2 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.84% | FLOPs estimator: workload-aware | logical tokens/s: 74176.2 | valid tokens/s: 74176.1 | physical tokens/s: 74701.2 | loss tokens/s: 58648.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.524489E-01 | lm-global-supervised-tokens: 1.326512E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.376973E-02 | lm-token-buffer-mib: 2.329251E+02 | global_load_balancing_loss: 9.998032E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11167.284 | params rms: 0.060467 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:24:13.173935 | finish at 2026-10-17 19:42:42 + [2026-10-15 14:22:15] iteration 1535/ 2384 | consumed samples: 49120 | elapsed time per iteration (ms): 226305.1 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.97% | FLOPs estimator: workload-aware | logical tokens/s: 74135.2 | valid tokens/s: 74135.2 | physical tokens/s: 75508.1 | loss tokens/s: 53669.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.187883E-01 | lm-global-supervised-tokens: 1.214573E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.290700E-02 | lm-token-buffer-mib: 2.369039E+02 | global_load_balancing_loss: 1.000015E+00 | loss scale: 1.0 | grad norm: 0.237 | params norm: 11167.247 | params rms: 0.060467 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:22:13.004972 | finish at 2026-10-17 19:44:28 + [2026-10-15 14:25:57] iteration 1536/ 2384 | consumed samples: 49152 | elapsed time per iteration (ms): 222347.7 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.55% | FLOPs estimator: workload-aware | logical tokens/s: 75454.7 | valid tokens/s: 75454.7 | physical tokens/s: 76278.1 | loss tokens/s: 57573.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.856784E-01 | lm-global-supervised-tokens: 1.280126E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.323561E-02 | lm-token-buffer-mib: 2.339804E+02 | global_load_balancing_loss: 9.996239E-01 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11167.209 | params rms: 0.060467 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:22:30.890541 | finish at 2026-10-17 18:48:28 + [2026-10-15 14:29:40] iteration 1537/ 2384 | consumed samples: 49184 | elapsed time per iteration (ms): 222532.4 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.63% | FLOPs estimator: workload-aware | logical tokens/s: 75392.2 | valid tokens/s: 75392.2 | physical tokens/s: 76091.7 | loss tokens/s: 59596.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.842818E-01 | lm-global-supervised-tokens: 1.326222E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.170606E-02 | lm-token-buffer-mib: 2.333602E+02 | global_load_balancing_loss: 9.997900E-01 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11167.170 | params rms: 0.060467 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:21:24.980068 | finish at 2026-10-17 18:51:05 + [2026-10-15 14:33:19] iteration 1538/ 2384 | consumed samples: 49216 | elapsed time per iteration (ms): 219617.6 | throughput per GPU (TFLOP/s/GPU): 18.8 | MFU 6.01% | FLOPs estimator: workload-aware | logical tokens/s: 76392.7 | valid tokens/s: 76392.7 | physical tokens/s: 77349.9 | loss tokens/s: 58721.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.606739E-01 | lm-global-supervised-tokens: 1.289625E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.176157E-02 | lm-token-buffer-mib: 2.356595E+02 | global_load_balancing_loss: 1.000156E+00 | loss scale: 1.0 | grad norm: 0.108 | params norm: 11167.128 | params rms: 0.060466 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 3:36:36.497033 | finish at 2026-10-17 18:09:56 + [2026-10-15 14:37:00] iteration 1539/ 2384 | consumed samples: 49248 | elapsed time per iteration (ms): 220585.7 | throughput per GPU (TFLOP/s/GPU): 27.0 | MFU 8.65% | FLOPs estimator: workload-aware | logical tokens/s: 76057.4 | valid tokens/s: 76057.4 | physical tokens/s: 76469.9 | loss tokens/s: 56833.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.237867E-01 | lm-global-supervised-tokens: 1.253654E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.251626E-02 | lm-token-buffer-mib: 2.339734E+02 | global_load_balancing_loss: 9.989996E-01 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11167.088 | params rms: 0.060466 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 3:46:34.941108 | finish at 2026-10-17 18:23:35 + [2026-10-15 14:41:03] iteration 1540/ 2384 | consumed samples: 49280 | elapsed time per iteration (ms): 243012.7 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.70% | FLOPs estimator: workload-aware | logical tokens/s: 69038.3 | valid tokens/s: 69038.3 | physical tokens/s: 69478.0 | loss tokens/s: 50238.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.909902E-01 | lm-global-supervised-tokens: 1.220856E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.632394E-02 | lm-token-buffer-mib: 2.327770E+02 | global_load_balancing_loss: 1.001908E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11167.047 | params rms: 0.060466 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:58:22.707197 | finish at 2026-10-17 23:39:26 + [2026-10-15 14:44:44] iteration 1541/ 2384 | consumed samples: 49312 | elapsed time per iteration (ms): 221251.1 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.52% | FLOPs estimator: workload-aware | logical tokens/s: 75828.7 | valid tokens/s: 75828.7 | physical tokens/s: 76774.5 | loss tokens/s: 58456.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.855507E-01 | lm-global-supervised-tokens: 1.293362E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.361589E-02 | lm-token-buffer-mib: 2.343214E+02 | global_load_balancing_loss: 1.001127E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11167.005 | params rms: 0.060466 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 3:48:34.656250 | finish at 2026-10-17 18:33:19 + [2026-10-15 14:48:25] iteration 1542/ 2384 | consumed samples: 49344 | elapsed time per iteration (ms): 220714.0 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.51% | FLOPs estimator: workload-aware | logical tokens/s: 76013.3 | valid tokens/s: 76013.3 | physical tokens/s: 76830.5 | loss tokens/s: 58364.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.218228E-01 | lm-global-supervised-tokens: 1.288191E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.777531E-02 | lm-token-buffer-mib: 2.334861E+02 | global_load_balancing_loss: 1.000384E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11166.962 | params rms: 0.060465 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 3:37:21.179557 | finish at 2026-10-17 18:25:46 + [2026-10-15 14:52:08] iteration 1543/ 2384 | consumed samples: 49376 | elapsed time per iteration (ms): 222707.6 | throughput per GPU (TFLOP/s/GPU): 19.2 | MFU 6.14% | FLOPs estimator: workload-aware | logical tokens/s: 75332.9 | valid tokens/s: 75332.9 | physical tokens/s: 76021.2 | loss tokens/s: 60061.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.865418E-01 | lm-global-supervised-tokens: 1.337615E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 6.521299E-02 | lm-token-buffer-mib: 2.341692E+02 | global_load_balancing_loss: 1.001560E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11166.919 | params rms: 0.060465 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:01:37.073853 | finish at 2026-10-17 18:53:45 + [2026-10-15 14:55:50] iteration 1544/ 2384 | consumed samples: 49408 | elapsed time per iteration (ms): 222089.5 | throughput per GPU (TFLOP/s/GPU): 19.3 | MFU 6.19% | FLOPs estimator: workload-aware | logical tokens/s: 75542.6 | valid tokens/s: 75542.6 | physical tokens/s: 76548.0 | loss tokens/s: 57993.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.214529E-01 | lm-global-supervised-tokens: 1.287973E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.300003E-02 | lm-token-buffer-mib: 2.365747E+02 | global_load_balancing_loss: 9.999098E-01 | loss scale: 1.0 | grad norm: 0.096 | params norm: 11166.876 | params rms: 0.060465 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 3:49:15.148916 | finish at 2026-10-17 18:45:05 + [2026-10-15 14:59:34] iteration 1545/ 2384 | consumed samples: 49440 | elapsed time per iteration (ms): 224308.0 | throughput per GPU (TFLOP/s/GPU): 25.2 | MFU 8.09% | FLOPs estimator: workload-aware | logical tokens/s: 74795.3 | valid tokens/s: 74795.3 | physical tokens/s: 75037.1 | loss tokens/s: 58410.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.909991E-01 | lm-global-supervised-tokens: 1.310185E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.563028E-02 | lm-token-buffer-mib: 2.328483E+02 | global_load_balancing_loss: 9.989529E-01 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11166.835 | params rms: 0.060465 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:16:34.439278 | finish at 2026-10-17 19:16:09 + [2026-10-15 15:03:19] iteration 1546/ 2384 | consumed samples: 49472 | elapsed time per iteration (ms): 224515.0 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.04% | FLOPs estimator: workload-aware | logical tokens/s: 74726.4 | valid tokens/s: 74726.4 | physical tokens/s: 75483.3 | loss tokens/s: 55288.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.786119E-01 | lm-global-supervised-tokens: 1.241306E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.104067E-02 | lm-token-buffer-mib: 2.350689E+02 | global_load_balancing_loss: 1.001202E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11166.798 | params rms: 0.060465 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:15:43.530529 | finish at 2026-10-17 19:19:02 + [2026-10-15 15:07:08] iteration 1547/ 2384 | consumed samples: 49504 | elapsed time per iteration (ms): 229357.1 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.37% | FLOPs estimator: workload-aware | logical tokens/s: 73148.8 | valid tokens/s: 73148.8 | physical tokens/s: 74411.5 | loss tokens/s: 55016.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.308621E-01 | lm-global-supervised-tokens: 1.261844E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.098169E-02 | lm-token-buffer-mib: 2.385493E+02 | global_load_balancing_loss: 1.002582E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11166.763 | params rms: 0.060464 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:19:31.882331 | finish at 2026-10-17 20:26:40 + [2026-10-15 15:10:52] iteration 1548/ 2384 | consumed samples: 49536 | elapsed time per iteration (ms): 223907.3 | throughput per GPU (TFLOP/s/GPU): 24.5 | MFU 7.85% | FLOPs estimator: workload-aware | logical tokens/s: 74929.3 | valid tokens/s: 74929.2 | physical tokens/s: 75248.6 | loss tokens/s: 55099.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.988708E-01 | lm-global-supervised-tokens: 1.233710E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.260407E-02 | lm-token-buffer-mib: 2.324931E+02 | global_load_balancing_loss: 1.001196E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11166.723 | params rms: 0.060464 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 3:59:46.502398 | finish at 2026-10-17 19:10:38 + [2026-10-15 15:14:35] iteration 1549/ 2384 | consumed samples: 49568 | elapsed time per iteration (ms): 222579.3 | throughput per GPU (TFLOP/s/GPU): 25.6 | MFU 8.19% | FLOPs estimator: workload-aware | logical tokens/s: 75376.2 | valid tokens/s: 75376.2 | physical tokens/s: 76152.4 | loss tokens/s: 57166.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.936522E-01 | lm-global-supervised-tokens: 1.272401E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.292804E-02 | lm-token-buffer-mib: 2.348355E+02 | global_load_balancing_loss: 1.000539E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11166.684 | params rms: 0.060464 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 3:37:33.694137 | finish at 2026-10-17 18:52:08 + [2026-10-15 15:18:37] iteration 1550/ 2384 | consumed samples: 49600 | elapsed time per iteration (ms): 242445.0 | throughput per GPU (TFLOP/s/GPU): 19.9 | MFU 6.39% | FLOPs estimator: workload-aware | logical tokens/s: 69199.9 | valid tokens/s: 69199.9 | physical tokens/s: 69622.5 | loss tokens/s: 47629.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.967690E-01 | lm-global-supervised-tokens: 1.154744E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.350502E-02 | lm-token-buffer-mib: 2.331215E+02 | global_load_balancing_loss: 9.996024E-01 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11166.651 | params rms: 0.060464 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:09:59.104294 | finish at 2026-10-17 23:28:36 + [2026-10-15 15:22:16] iteration 1551/ 2384 | consumed samples: 49632 | elapsed time per iteration (ms): 219417.9 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.34% | FLOPs estimator: workload-aware | logical tokens/s: 76462.3 | valid tokens/s: 76462.3 | physical tokens/s: 77301.3 | loss tokens/s: 56220.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.330567E-01 | lm-global-supervised-tokens: 1.233567E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.337673E-02 | lm-token-buffer-mib: 2.337776E+02 | global_load_balancing_loss: 9.988915E-01 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11166.618 | params rms: 0.060464 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:46:15.098259 | finish at 2026-10-17 18:08:31 + [2026-10-15 15:25:57] iteration 1552/ 2384 | consumed samples: 49664 | elapsed time per iteration (ms): 221000.7 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.01% | FLOPs estimator: workload-aware | logical tokens/s: 75914.7 | valid tokens/s: 75914.7 | physical tokens/s: 76396.4 | loss tokens/s: 55579.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.792568E-01 | lm-global-supervised-tokens: 1.228304E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.091434E-02 | lm-token-buffer-mib: 2.334605E+02 | global_load_balancing_loss: 9.984958E-01 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11166.587 | params rms: 0.060463 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 3:04:32.577835 | finish at 2026-10-17 18:30:30 + [2026-10-15 15:29:38] iteration 1553/ 2384 | consumed samples: 49696 | elapsed time per iteration (ms): 220917.9 | throughput per GPU (TFLOP/s/GPU): 19.5 | MFU 6.26% | FLOPs estimator: workload-aware | logical tokens/s: 75943.2 | valid tokens/s: 75943.2 | physical tokens/s: 76941.2 | loss tokens/s: 57965.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.490361E-01 | lm-global-supervised-tokens: 1.280558E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.242990E-02 | lm-token-buffer-mib: 2.356334E+02 | global_load_balancing_loss: 9.994065E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11166.554 | params rms: 0.060463 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:59:42.804690 | finish at 2026-10-17 18:29:21 + [2026-10-15 15:33:22] iteration 1554/ 2384 | consumed samples: 49728 | elapsed time per iteration (ms): 223632.2 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.55% | FLOPs estimator: workload-aware | logical tokens/s: 75021.2 | valid tokens/s: 75021.2 | physical tokens/s: 75797.9 | loss tokens/s: 57804.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.673818E-01 | lm-global-supervised-tokens: 1.292684E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.486348E-02 | lm-token-buffer-mib: 2.347209E+02 | global_load_balancing_loss: 1.000253E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11166.518 | params rms: 0.060463 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 3:33:34.749749 | finish at 2026-10-17 19:06:57 + [2026-10-15 15:37:03] iteration 1555/ 2384 | consumed samples: 49760 | elapsed time per iteration (ms): 220647.0 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.39% | FLOPs estimator: workload-aware | logical tokens/s: 76036.3 | valid tokens/s: 76036.3 | physical tokens/s: 76709.7 | loss tokens/s: 58432.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.847624E-01 | lm-global-supervised-tokens: 1.289284E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.493057E-02 | lm-token-buffer-mib: 2.325037E+02 | global_load_balancing_loss: 1.002352E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11166.482 | params rms: 0.060463 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:48:36.390337 | finish at 2026-10-17 18:25:39 + [2026-10-15 15:40:48] iteration 1556/ 2384 | consumed samples: 49792 | elapsed time per iteration (ms): 225536.1 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.20% | FLOPs estimator: workload-aware | logical tokens/s: 74388.0 | valid tokens/s: 74388.0 | physical tokens/s: 74950.9 | loss tokens/s: 57102.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.645027E-01 | lm-global-supervised-tokens: 1.287868E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.200202E-02 | lm-token-buffer-mib: 2.349961E+02 | global_load_balancing_loss: 1.000127E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11166.446 | params rms: 0.060463 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 3:52:23.887568 | finish at 2026-10-17 19:33:12 + [2026-10-15 15:44:41] iteration 1557/ 2384 | consumed samples: 49824 | elapsed time per iteration (ms): 232577.7 | throughput per GPU (TFLOP/s/GPU): 24.5 | MFU 7.86% | FLOPs estimator: workload-aware | logical tokens/s: 72135.7 | valid tokens/s: 72135.7 | physical tokens/s: 73984.6 | loss tokens/s: 54843.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.584392E-01 | lm-global-supervised-tokens: 1.275545E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.073390E-02 | lm-token-buffer-mib: 2.412458E+02 | global_load_balancing_loss: 1.000968E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11166.412 | params rms: 0.060462 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:25:41.788182 | finish at 2026-10-17 21:10:22 + [2026-10-15 15:48:21] iteration 1558/ 2384 | consumed samples: 49856 | elapsed time per iteration (ms): 219968.3 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.07% | FLOPs estimator: workload-aware | logical tokens/s: 76271.0 | valid tokens/s: 76271.0 | physical tokens/s: 76995.8 | loss tokens/s: 54567.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.531222E-01 | lm-global-supervised-tokens: 1.200304E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.629366E-02 | lm-token-buffer-mib: 2.342145E+02 | global_load_balancing_loss: 1.001491E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11166.377 | params rms: 0.060462 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:28:13.799541 | finish at 2026-10-17 18:16:34 + [2026-10-15 15:52:04] iteration 1559/ 2384 | consumed samples: 49888 | elapsed time per iteration (ms): 222991.7 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.10% | FLOPs estimator: workload-aware | logical tokens/s: 75236.8 | valid tokens/s: 75236.8 | physical tokens/s: 75999.3 | loss tokens/s: 60621.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.398402E-01 | lm-global-supervised-tokens: 1.351797E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.087792E-02 | lm-token-buffer-mib: 2.332265E+02 | global_load_balancing_loss: 1.001302E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11166.343 | params rms: 0.060462 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 3:06:08.114090 | finish at 2026-10-17 18:58:12 + [2026-10-15 15:56:10] iteration 1560/ 2384 | consumed samples: 49920 | elapsed time per iteration (ms): 246242.4 | throughput per GPU (TFLOP/s/GPU): 18.9 | MFU 6.06% | FLOPs estimator: workload-aware | logical tokens/s: 68132.8 | valid tokens/s: 68132.8 | physical tokens/s: 68896.8 | loss tokens/s: 54065.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.197434E-01 | lm-global-supervised-tokens: 1.331318E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.561857E-02 | lm-token-buffer-mib: 2.337125E+02 | global_load_balancing_loss: 1.000135E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11166.312 | params rms: 0.060462 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:21:43.703360 | finish at 2026-10-18 00:17:54 +times across ranks (ms): + save-checkpoint: + rank 0: 25025.90 + rank 1: 25025.92 + rank 2: 25025.93 + rank 3: 25025.93 + rank 4: 25025.90 + rank 5: 25025.90 + rank 6: 25025.89 + rank 7: 25025.88 + rank 8: 25025.77 + rank 9: 25025.81 + rank 10: 25025.82 + rank 11: 25025.77 + rank 12: 25025.80 + rank 13: 25025.77 + rank 14: 25025.79 + rank 15: 25025.79 + rank 16: 25026.29 + rank 17: 25026.30 + rank 18: 25026.31 + rank 19: 25026.27 + rank 20: 25026.32 + rank 21: 25026.26 + rank 22: 25026.32 + rank 23: 25026.28 + rank 24: 25026.17 + rank 25: 25026.18 + rank 26: 25026.04 + rank 27: 25026.37 + rank 28: 25026.11 + rank 29: 25026.81 + rank 30: 25026.20 + rank 31: 25025.90 + rank 32: 25025.89 + rank 33: 25026.08 + rank 34: 25025.92 + rank 35: 25025.92 + rank 36: 25025.86 + rank 37: 25025.94 + rank 38: 25025.86 + rank 39: 25025.95 + rank 40: 25025.94 + rank 41: 25025.90 + rank 42: 25025.91 + rank 43: 25025.95 + rank 44: 25025.88 + rank 45: 25025.88 + rank 46: 25025.88 + rank 47: 25025.89 + rank 48: 25026.12 + rank 49: 25026.17 + rank 50: 25026.14 + rank 51: 25026.21 + rank 52: 25026.17 + rank 53: 25026.28 + rank 54: 25026.23 + rank 55: 25026.24 + rank 56: 25026.26 + rank 57: 25026.29 + rank 58: 25026.29 + rank 59: 25026.28 + rank 60: 25026.32 + rank 61: 25026.26 + rank 62: 25026.31 + rank 63: 25026.26 + [2026-10-15 16:00:14] iteration 1561/ 2384 | consumed samples: 49952 | elapsed time per iteration (ms): 219235.6 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.14% | FLOPs estimator: workload-aware | logical tokens/s: 76525.7 | valid tokens/s: 76525.7 | physical tokens/s: 77617.7 | loss tokens/s: 57362.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.863064E-01 | lm-global-supervised-tokens: 1.257593E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 5.295646E-01 | lm-token-buffer-mib: 2.365790E+02 | global_load_balancing_loss: 1.001211E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11166.283 | params rms: 0.060462 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:07:10.924893 | finish at 2026-10-17 18:07:25 + [2026-10-15 16:04:02] iteration 1562/ 2384 | consumed samples: 49984 | elapsed time per iteration (ms): 227876.2 | throughput per GPU (TFLOP/s/GPU): 18.5 | MFU 5.93% | FLOPs estimator: workload-aware | logical tokens/s: 73624.2 | valid tokens/s: 73624.2 | physical tokens/s: 74480.6 | loss tokens/s: 59122.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.307088E-01 | lm-global-supervised-tokens: 1.347253E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.747115E-02 | lm-token-buffer-mib: 2.359142E+02 | global_load_balancing_loss: 1.000261E+00 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11166.251 | params rms: 0.060462 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:01:54.205991 | finish at 2026-10-17 20:05:56 + [2026-10-15 16:07:43] iteration 1563/ 2384 | consumed samples: 50016 | elapsed time per iteration (ms): 220885.8 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.72% | FLOPs estimator: workload-aware | logical tokens/s: 75954.1 | valid tokens/s: 75954.1 | physical tokens/s: 76144.0 | loss tokens/s: 55256.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.463203E-01 | lm-global-supervised-tokens: 1.220530E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.181765E-02 | lm-token-buffer-mib: 2.327719E+02 | global_load_balancing_loss: 9.995975E-01 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11166.219 | params rms: 0.060461 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:22:27.246207 | finish at 2026-10-17 18:30:10 + [2026-10-15 16:11:27] iteration 1564/ 2384 | consumed samples: 50048 | elapsed time per iteration (ms): 223865.8 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.46% | FLOPs estimator: workload-aware | logical tokens/s: 74943.0 | valid tokens/s: 74943.0 | physical tokens/s: 75854.9 | loss tokens/s: 58039.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.673638E-01 | lm-global-supervised-tokens: 1.299297E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.671735E-02 | lm-token-buffer-mib: 2.359202E+02 | global_load_balancing_loss: 1.001746E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11166.186 | params rms: 0.060461 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:59:29.927573 | finish at 2026-10-17 19:10:57 + [2026-10-15 16:15:08] iteration 1565/ 2384 | consumed samples: 50080 | elapsed time per iteration (ms): 220986.7 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.04% | FLOPs estimator: workload-aware | logical tokens/s: 75919.4 | valid tokens/s: 75919.4 | physical tokens/s: 76824.0 | loss tokens/s: 57018.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.013259E-01 | lm-global-supervised-tokens: 1.260034E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.938531E-02 | lm-token-buffer-mib: 2.343030E+02 | global_load_balancing_loss: 9.997008E-01 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11166.157 | params rms: 0.060461 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:16:28.106957 | finish at 2026-10-17 18:31:36 + [2026-10-15 16:18:47] iteration 1566/ 2384 | consumed samples: 50112 | elapsed time per iteration (ms): 219410.9 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.72% | FLOPs estimator: workload-aware | logical tokens/s: 76464.8 | valid tokens/s: 76464.8 | physical tokens/s: 77067.2 | loss tokens/s: 55576.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.262597E-01 | lm-global-supervised-tokens: 1.219416E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 6.283648E-02 | lm-token-buffer-mib: 2.335948E+02 | global_load_balancing_loss: 9.998002E-01 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11166.127 | params rms: 0.060461 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:51:18.109079 | finish at 2026-10-17 18:10:05 + [2026-10-15 16:22:28] iteration 1567/ 2384 | consumed samples: 50144 | elapsed time per iteration (ms): 220350.0 | throughput per GPU (TFLOP/s/GPU): 23.4 | MFU 7.51% | FLOPs estimator: workload-aware | logical tokens/s: 76138.9 | valid tokens/s: 76138.9 | physical tokens/s: 77038.8 | loss tokens/s: 57028.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.705429E-01 | lm-global-supervised-tokens: 1.256627E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.051386E-02 | lm-token-buffer-mib: 2.325976E+02 | global_load_balancing_loss: 9.990817E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11166.097 | params rms: 0.060461 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:00:25.940377 | finish at 2026-10-17 18:22:54 + [2026-10-15 16:26:11] iteration 1568/ 2384 | consumed samples: 50176 | elapsed time per iteration (ms): 223560.7 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.68% | FLOPs estimator: workload-aware | logical tokens/s: 75045.4 | valid tokens/s: 75045.4 | physical tokens/s: 76206.6 | loss tokens/s: 60283.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.975913E-01 | lm-global-supervised-tokens: 1.347708E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.415329E-02 | lm-token-buffer-mib: 2.354988E+02 | global_load_balancing_loss: 1.000035E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11166.068 | params rms: 0.060461 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:40:25.558971 | finish at 2026-10-17 19:06:37 + [2026-10-15 16:29:55] iteration 1569/ 2384 | consumed samples: 50208 | elapsed time per iteration (ms): 223987.9 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.87% | FLOPs estimator: workload-aware | logical tokens/s: 74902.2 | valid tokens/s: 74902.2 | physical tokens/s: 75944.1 | loss tokens/s: 54717.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.553680E-01 | lm-global-supervised-tokens: 1.225614E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.423855E-02 | lm-token-buffer-mib: 2.359724E+02 | global_load_balancing_loss: 9.996910E-01 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11166.040 | params rms: 0.060460 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:42:30.130160 | finish at 2026-10-17 19:12:25 + [2026-10-15 16:34:00] iteration 1570/ 2384 | consumed samples: 50240 | elapsed time per iteration (ms): 244771.9 | throughput per GPU (TFLOP/s/GPU): 17.8 | MFU 5.72% | FLOPs estimator: workload-aware | logical tokens/s: 68542.1 | valid tokens/s: 68542.1 | physical tokens/s: 68852.9 | loss tokens/s: 52750.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.685548E-01 | lm-global-supervised-tokens: 1.291174E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.220912E-02 | lm-token-buffer-mib: 2.331408E+02 | global_load_balancing_loss: 1.001086E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11166.012 | params rms: 0.060460 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 7:20:44.358960 | finish at 2026-10-17 23:54:44 + [2026-10-15 16:37:44] iteration 1571/ 2384 | consumed samples: 50272 | elapsed time per iteration (ms): 223683.9 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.78% | FLOPs estimator: workload-aware | logical tokens/s: 75004.1 | valid tokens/s: 75004.1 | physical tokens/s: 75789.5 | loss tokens/s: 56927.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.426255E-01 | lm-global-supervised-tokens: 1.273381E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.606880E-02 | lm-token-buffer-mib: 2.349763E+02 | global_load_balancing_loss: 1.000838E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11165.987 | params rms: 0.060460 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:30:55.035607 | finish at 2026-10-17 19:08:39 + [2026-10-15 16:41:26] iteration 1572/ 2384 | consumed samples: 50304 | elapsed time per iteration (ms): 222696.7 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.05% | FLOPs estimator: workload-aware | logical tokens/s: 75336.5 | valid tokens/s: 75336.5 | physical tokens/s: 75820.9 | loss tokens/s: 56600.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.422894E-01 | lm-global-supervised-tokens: 1.260469E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.932163E-02 | lm-token-buffer-mib: 2.333836E+02 | global_load_balancing_loss: 1.001583E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11165.960 | params rms: 0.060460 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:13:49.747969 | finish at 2026-10-17 18:55:16 + [2026-10-15 16:45:05] iteration 1573/ 2384 | consumed samples: 50336 | elapsed time per iteration (ms): 218428.9 | throughput per GPU (TFLOP/s/GPU): 19.2 | MFU 6.14% | FLOPs estimator: workload-aware | logical tokens/s: 76808.5 | valid tokens/s: 76808.5 | physical tokens/s: 78096.0 | loss tokens/s: 59149.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.627133E-01 | lm-global-supervised-tokens: 1.291986E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.253088E-02 | lm-token-buffer-mib: 2.328956E+02 | global_load_balancing_loss: 1.000458E+00 | loss scale: 1.0 | grad norm: 0.251 | params norm: 11165.933 | params rms: 0.060460 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:12:25.851051 | finish at 2026-10-17 17:57:31 + [2026-10-15 16:48:51] iteration 1574/ 2384 | consumed samples: 50368 | elapsed time per iteration (ms): 226662.5 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.25% | FLOPs estimator: workload-aware | logical tokens/s: 74018.4 | valid tokens/s: 74018.4 | physical tokens/s: 74933.6 | loss tokens/s: 54706.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.466240E-01 | lm-global-supervised-tokens: 1.239979E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.249125E-02 | lm-token-buffer-mib: 2.362853E+02 | global_load_balancing_loss: 1.000704E+00 | loss scale: 1.0 | grad norm: 0.190 | params norm: 11165.904 | params rms: 0.060460 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:59:56.603680 | finish at 2026-10-17 19:48:48 + [2026-10-15 16:52:33] iteration 1575/ 2384 | consumed samples: 50400 | elapsed time per iteration (ms): 221695.9 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.06% | FLOPs estimator: workload-aware | logical tokens/s: 75676.5 | valid tokens/s: 75676.5 | physical tokens/s: 76387.4 | loss tokens/s: 56883.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.014980E-01 | lm-global-supervised-tokens: 1.261092E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.256346E-02 | lm-token-buffer-mib: 2.341684E+02 | global_load_balancing_loss: 1.000181E+00 | loss scale: 1.0 | grad norm: 0.160 | params norm: 11165.876 | params rms: 0.060460 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:49:11.980177 | finish at 2026-10-17 18:41:45 + [2026-10-15 16:56:17] iteration 1576/ 2384 | consumed samples: 50432 | elapsed time per iteration (ms): 224151.3 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.03% | FLOPs estimator: workload-aware | logical tokens/s: 74847.5 | valid tokens/s: 74847.5 | physical tokens/s: 75168.9 | loss tokens/s: 52500.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.436435E-01 | lm-global-supervised-tokens: 1.176801E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.448944E-02 | lm-token-buffer-mib: 2.322408E+02 | global_load_balancing_loss: 1.000570E+00 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11165.847 | params rms: 0.060459 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:18:34.241693 | finish at 2026-10-17 19:14:52 + [2026-10-15 17:00:00] iteration 1577/ 2384 | consumed samples: 50464 | elapsed time per iteration (ms): 222987.0 | throughput per GPU (TFLOP/s/GPU): 18.2 | MFU 5.82% | FLOPs estimator: workload-aware | logical tokens/s: 75238.4 | valid tokens/s: 75238.4 | physical tokens/s: 76041.6 | loss tokens/s: 56506.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.083463E-01 | lm-global-supervised-tokens: 1.260028E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.248651E-02 | lm-token-buffer-mib: 2.352969E+02 | global_load_balancing_loss: 1.000003E+00 | loss scale: 1.0 | grad norm: 0.169 | params norm: 11165.817 | params rms: 0.060459 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:59:10.480323 | finish at 2026-10-17 18:59:11 + [2026-10-15 17:03:39] iteration 1578/ 2384 | consumed samples: 50496 | elapsed time per iteration (ms): 219032.8 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.09% | FLOPs estimator: workload-aware | logical tokens/s: 76596.7 | valid tokens/s: 76596.7 | physical tokens/s: 76870.3 | loss tokens/s: 56126.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.408251E-01 | lm-global-supervised-tokens: 1.229355E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.176261E-02 | lm-token-buffer-mib: 2.334162E+02 | global_load_balancing_loss: 9.996058E-01 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11165.790 | params rms: 0.060459 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:02:20.450795 | finish at 2026-10-17 18:06:00 + [2026-10-15 17:07:24] iteration 1579/ 2384 | consumed samples: 50528 | elapsed time per iteration (ms): 224880.8 | throughput per GPU (TFLOP/s/GPU): 24.0 | MFU 7.69% | FLOPs estimator: workload-aware | logical tokens/s: 74604.8 | valid tokens/s: 74604.8 | physical tokens/s: 75735.6 | loss tokens/s: 53739.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.472960E-01 | lm-global-supervised-tokens: 1.208501E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.925015E-02 | lm-token-buffer-mib: 2.341057E+02 | global_load_balancing_loss: 1.000189E+00 | loss scale: 1.0 | grad norm: 0.176 | params norm: 11165.760 | params rms: 0.060459 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:17:09.035946 | finish at 2026-10-17 19:24:33 + [2026-10-15 17:11:27] iteration 1580/ 2384 | consumed samples: 50560 | elapsed time per iteration (ms): 242970.2 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.52% | FLOPs estimator: workload-aware | logical tokens/s: 69050.4 | valid tokens/s: 69050.4 | physical tokens/s: 69679.3 | loss tokens/s: 54590.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.026244E-01 | lm-global-supervised-tokens: 1.326393E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.244761E-02 | lm-token-buffer-mib: 2.332308E+02 | global_load_balancing_loss: 9.995700E-01 | loss scale: 1.0 | grad norm: 0.117 | params norm: 11165.729 | params rms: 0.060459 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:15:48.061360 | finish at 2026-10-17 23:27:15 + [2026-10-15 17:15:08] iteration 1581/ 2384 | consumed samples: 50592 | elapsed time per iteration (ms): 220610.8 | throughput per GPU (TFLOP/s/GPU): 24.5 | MFU 7.84% | FLOPs estimator: workload-aware | logical tokens/s: 76048.8 | valid tokens/s: 76048.8 | physical tokens/s: 76506.4 | loss tokens/s: 56780.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.600031E-01 | lm-global-supervised-tokens: 1.252638E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.493091E-02 | lm-token-buffer-mib: 2.328573E+02 | global_load_balancing_loss: 1.000193E+00 | loss scale: 1.0 | grad norm: 0.122 | params norm: 11165.698 | params rms: 0.060459 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:12:30.486207 | finish at 2026-10-17 18:27:38 + [2026-10-15 17:18:49] iteration 1582/ 2384 | consumed samples: 50624 | elapsed time per iteration (ms): 221445.1 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.98% | FLOPs estimator: workload-aware | logical tokens/s: 75762.3 | valid tokens/s: 75762.3 | physical tokens/s: 76124.5 | loss tokens/s: 54031.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.152682E-01 | lm-global-supervised-tokens: 1.196509E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.667814E-02 | lm-token-buffer-mib: 2.333215E+02 | global_load_balancing_loss: 1.001034E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11165.667 | params rms: 0.060458 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:19:58.984214 | finish at 2026-10-17 18:38:48 + [2026-10-15 17:22:35] iteration 1583/ 2384 | consumed samples: 50656 | elapsed time per iteration (ms): 225519.5 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.24% | FLOPs estimator: workload-aware | logical tokens/s: 74393.5 | valid tokens/s: 74393.5 | physical tokens/s: 75629.8 | loss tokens/s: 57482.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.457957E-01 | lm-global-supervised-tokens: 1.296332E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.092774E-02 | lm-token-buffer-mib: 2.361875E+02 | global_load_balancing_loss: 1.000539E+00 | loss scale: 1.0 | grad norm: 0.111 | params norm: 11165.633 | params rms: 0.060458 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:10:41.100225 | finish at 2026-10-17 19:33:16 + [2026-10-15 17:26:20] iteration 1584/ 2384 | consumed samples: 50688 | elapsed time per iteration (ms): 225458.6 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.97% | FLOPs estimator: workload-aware | logical tokens/s: 74413.7 | valid tokens/s: 74413.7 | physical tokens/s: 75488.2 | loss tokens/s: 57755.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.621272E-01 | lm-global-supervised-tokens: 1.302138E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.873605E-02 | lm-token-buffer-mib: 2.327695E+02 | global_load_balancing_loss: 1.000466E+00 | loss scale: 1.0 | grad norm: 0.165 | params norm: 11165.600 | params rms: 0.060458 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:06:06.871262 | finish at 2026-10-17 19:32:27 + [2026-10-15 17:30:00] iteration 1585/ 2384 | consumed samples: 50720 | elapsed time per iteration (ms): 220238.8 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.98% | FLOPs estimator: workload-aware | logical tokens/s: 76177.3 | valid tokens/s: 76177.3 | physical tokens/s: 76769.0 | loss tokens/s: 59970.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.842898E-01 | lm-global-supervised-tokens: 1.320792E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.177776E-02 | lm-token-buffer-mib: 2.334052E+02 | global_load_balancing_loss: 1.001072E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11165.564 | params rms: 0.060458 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:52:50.808478 | finish at 2026-10-17 18:22:51 + [2026-10-15 17:33:46] iteration 1586/ 2384 | consumed samples: 50752 | elapsed time per iteration (ms): 225160.3 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.35% | FLOPs estimator: workload-aware | logical tokens/s: 74512.3 | valid tokens/s: 74512.3 | physical tokens/s: 75628.3 | loss tokens/s: 54288.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.237940E-01 | lm-global-supervised-tokens: 1.222364E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.014279E-02 | lm-token-buffer-mib: 2.339729E+02 | global_load_balancing_loss: 1.001660E+00 | loss scale: 1.0 | grad norm: 0.263 | params norm: 11165.530 | params rms: 0.060458 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:54:37.884976 | finish at 2026-10-17 19:28:24 + [2026-10-15 17:37:30] iteration 1587/ 2384 | consumed samples: 50784 | elapsed time per iteration (ms): 224390.3 | throughput per GPU (TFLOP/s/GPU): 23.8 | MFU 7.62% | FLOPs estimator: workload-aware | logical tokens/s: 74767.9 | valid tokens/s: 74767.9 | physical tokens/s: 75493.3 | loss tokens/s: 58818.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.908926E-01 | lm-global-supervised-tokens: 1.319826E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.476365E-02 | lm-token-buffer-mib: 2.349423E+02 | global_load_balancing_loss: 9.986909E-01 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11165.497 | params rms: 0.060458 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:40:39.043856 | finish at 2026-10-17 19:18:09 + [2026-10-15 17:41:17] iteration 1588/ 2384 | consumed samples: 50816 | elapsed time per iteration (ms): 227438.5 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.67% | FLOPs estimator: workload-aware | logical tokens/s: 73765.8 | valid tokens/s: 73765.8 | physical tokens/s: 74346.2 | loss tokens/s: 57880.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.784367E-01 | lm-global-supervised-tokens: 1.316427E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.237570E-02 | lm-token-buffer-mib: 2.344983E+02 | global_load_balancing_loss: 9.997101E-01 | loss scale: 1.0 | grad norm: 0.308 | params norm: 11165.463 | params rms: 0.060457 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:17:21.006847 | finish at 2026-10-17 19:58:38 + [2026-10-15 17:45:52] iteration 1589/ 2384 | consumed samples: 50848 | elapsed time per iteration (ms): 274510.2 | throughput per GPU (TFLOP/s/GPU): 18.2 | MFU 5.84% | FLOPs estimator: workload-aware | logical tokens/s: 61116.8 | valid tokens/s: 61116.8 | physical tokens/s: 62811.2 | loss tokens/s: 45099.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.639816E-01 | lm-global-supervised-tokens: 1.238019E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.155230E-02 | lm-token-buffer-mib: 2.432141E+02 | global_load_balancing_loss: 9.993586E-01 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11165.429 | params rms: 0.060457 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:37:15.623235 | finish at 2026-10-18 06:23:08 + [2026-10-15 17:49:56] iteration 1590/ 2384 | consumed samples: 50880 | elapsed time per iteration (ms): 244264.9 | throughput per GPU (TFLOP/s/GPU): 18.0 | MFU 5.78% | FLOPs estimator: workload-aware | logical tokens/s: 68684.5 | valid tokens/s: 68684.5 | physical tokens/s: 69160.6 | loss tokens/s: 52870.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.168607E-01 | lm-global-supervised-tokens: 1.291432E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.639356E-02 | lm-token-buffer-mib: 2.332891E+02 | global_load_balancing_loss: 1.000388E+00 | loss scale: 1.0 | grad norm: 0.550 | params norm: 11165.398 | params rms: 0.060457 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:52:26.327736 | finish at 2026-10-17 23:42:23 + [2026-10-15 17:53:36] iteration 1591/ 2384 | consumed samples: 50912 | elapsed time per iteration (ms): 219288.1 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.23% | FLOPs estimator: workload-aware | logical tokens/s: 76507.6 | valid tokens/s: 76507.5 | physical tokens/s: 77184.1 | loss tokens/s: 59326.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.647177E-01 | lm-global-supervised-tokens: 1.300966E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.459299E-02 | lm-token-buffer-mib: 2.342353E+02 | global_load_balancing_loss: 9.997951E-01 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11165.367 | params rms: 0.060457 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:18:15.501495 | finish at 2026-10-17 18:11:51 + [2026-10-15 17:57:16] iteration 1592/ 2384 | consumed samples: 50944 | elapsed time per iteration (ms): 220428.2 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.24% | FLOPs estimator: workload-aware | logical tokens/s: 76111.8 | valid tokens/s: 76111.8 | physical tokens/s: 76855.8 | loss tokens/s: 59604.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.633781E-01 | lm-global-supervised-tokens: 1.313841E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.713162E-02 | lm-token-buffer-mib: 2.344541E+02 | global_load_balancing_loss: 1.001594E+00 | loss scale: 1.0 | grad norm: 0.212 | params norm: 11165.333 | params rms: 0.060457 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:29:39.109858 | finish at 2026-10-17 18:26:55 + [2026-10-15 18:00:56] iteration 1593/ 2384 | consumed samples: 50976 | elapsed time per iteration (ms): 219699.2 | throughput per GPU (TFLOP/s/GPU): 18.2 | MFU 5.82% | FLOPs estimator: workload-aware | logical tokens/s: 76364.4 | valid tokens/s: 76364.4 | physical tokens/s: 76735.9 | loss tokens/s: 53644.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.895811E-01 | lm-global-supervised-tokens: 1.178571E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.813805E-02 | lm-token-buffer-mib: 2.327834E+02 | global_load_balancing_loss: 1.001131E+00 | loss scale: 1.0 | grad norm: 0.217 | params norm: 11165.297 | params rms: 0.060456 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:16:22.082220 | finish at 2026-10-17 18:17:18 + [2026-10-15 18:04:36] iteration 1594/ 2384 | consumed samples: 51008 | elapsed time per iteration (ms): 220305.0 | throughput per GPU (TFLOP/s/GPU): 24.0 | MFU 7.71% | FLOPs estimator: workload-aware | logical tokens/s: 76154.4 | valid tokens/s: 76154.4 | physical tokens/s: 76392.1 | loss tokens/s: 57367.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.196276E-01 | lm-global-supervised-tokens: 1.263839E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.601098E-02 | lm-token-buffer-mib: 2.328687E+02 | global_load_balancing_loss: 1.001791E+00 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11165.261 | params rms: 0.060456 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:20:40.972655 | finish at 2026-10-17 18:25:17 + [2026-10-15 18:08:18] iteration 1595/ 2384 | consumed samples: 51040 | elapsed time per iteration (ms): 222156.6 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.02% | FLOPs estimator: workload-aware | logical tokens/s: 75519.4 | valid tokens/s: 75519.4 | physical tokens/s: 77236.2 | loss tokens/s: 58940.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.067629E-01 | lm-global-supervised-tokens: 1.309399E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.686099E-02 | lm-token-buffer-mib: 2.365189E+02 | global_load_balancing_loss: 1.000317E+00 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11165.225 | params rms: 0.060456 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:41:21.571319 | finish at 2026-10-17 18:49:40 + [2026-10-15 18:12:03] iteration 1596/ 2384 | consumed samples: 51072 | elapsed time per iteration (ms): 225180.8 | throughput per GPU (TFLOP/s/GPU): 24.6 | MFU 7.88% | FLOPs estimator: workload-aware | logical tokens/s: 74505.4 | valid tokens/s: 74505.4 | physical tokens/s: 74995.8 | loss tokens/s: 56694.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.261151E-01 | lm-global-supervised-tokens: 1.276643E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.226979E-02 | lm-token-buffer-mib: 2.323839E+02 | global_load_balancing_loss: 9.992103E-01 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11165.189 | params rms: 0.060456 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:17:22.488593 | finish at 2026-10-17 19:29:26 + [2026-10-15 18:15:45] iteration 1597/ 2384 | consumed samples: 51104 | elapsed time per iteration (ms): 221395.3 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.59% | FLOPs estimator: workload-aware | logical tokens/s: 75779.3 | valid tokens/s: 75779.2 | physical tokens/s: 76330.7 | loss tokens/s: 59384.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.852144E-01 | lm-global-supervised-tokens: 1.314733E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.279142E-02 | lm-token-buffer-mib: 2.339742E+02 | global_load_balancing_loss: 9.989921E-01 | loss scale: 1.0 | grad norm: 0.107 | params norm: 11165.155 | params rms: 0.060456 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:23:58.119043 | finish at 2026-10-17 18:39:43 + [2026-10-15 18:19:30] iteration 1598/ 2384 | consumed samples: 51136 | elapsed time per iteration (ms): 224949.1 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.90% | FLOPs estimator: workload-aware | logical tokens/s: 74582.1 | valid tokens/s: 74582.1 | physical tokens/s: 75168.1 | loss tokens/s: 55417.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.432132E-01 | lm-global-supervised-tokens: 1.246613E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.421487E-02 | lm-token-buffer-mib: 2.344524E+02 | global_load_balancing_loss: 9.996157E-01 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11165.119 | params rms: 0.060455 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:06:49.953635 | finish at 2026-10-17 19:26:20 + [2026-10-15 18:23:15] iteration 1599/ 2384 | consumed samples: 51168 | elapsed time per iteration (ms): 225372.6 | throughput per GPU (TFLOP/s/GPU): 24.2 | MFU 7.77% | FLOPs estimator: workload-aware | logical tokens/s: 74442.0 | valid tokens/s: 74442.0 | physical tokens/s: 75005.0 | loss tokens/s: 55024.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.733060E-01 | lm-global-supervised-tokens: 1.240108E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.651550E-02 | lm-token-buffer-mib: 2.328157E+02 | global_load_balancing_loss: 1.000377E+00 | loss scale: 1.0 | grad norm: 0.189 | params norm: 11165.081 | params rms: 0.060455 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:08:37.493308 | finish at 2026-10-17 19:31:53 + [2026-10-15 18:27:18] iteration 1600/ 2384 | consumed samples: 51200 | elapsed time per iteration (ms): 242530.1 | throughput per GPU (TFLOP/s/GPU): 17.7 | MFU 5.66% | FLOPs estimator: workload-aware | logical tokens/s: 69175.7 | valid tokens/s: 69175.7 | physical tokens/s: 69991.8 | loss tokens/s: 53836.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.655195E-01 | lm-global-supervised-tokens: 1.305686E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.394689E-02 | lm-token-buffer-mib: 2.329082E+02 | global_load_balancing_loss: 1.000355E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11165.041 | params rms: 0.060455 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:49:03.571754 | finish at 2026-10-17 23:16:21 +times across ranks (ms): + save-checkpoint: + rank 0: 24702.74 + rank 1: 24702.74 + rank 2: 24702.80 + rank 3: 24702.76 + rank 4: 24702.73 + rank 5: 24702.76 + rank 6: 24702.76 + rank 7: 24702.77 + rank 8: 24703.34 + rank 9: 24703.35 + rank 10: 24703.35 + rank 11: 24703.35 + rank 12: 24703.33 + rank 13: 24703.51 + rank 14: 24703.31 + rank 15: 24703.33 + rank 16: 24702.82 + rank 17: 24702.81 + rank 18: 24702.81 + rank 19: 24702.81 + rank 20: 24702.79 + rank 21: 24702.80 + rank 22: 24702.82 + rank 23: 24702.82 + rank 24: 24703.05 + rank 25: 24703.07 + rank 26: 24703.10 + rank 27: 24703.02 + rank 28: 24703.04 + rank 29: 24703.07 + rank 30: 24703.04 + rank 31: 24703.35 + rank 32: 24702.92 + rank 33: 24702.89 + rank 34: 24702.92 + rank 35: 24702.97 + rank 36: 24702.82 + rank 37: 24702.86 + rank 38: 24702.79 + rank 39: 24702.81 + rank 40: 24702.86 + rank 41: 24702.84 + rank 42: 24702.85 + rank 43: 24702.84 + rank 44: 24702.83 + rank 45: 24704.12 + rank 46: 24702.82 + rank 47: 24702.82 + rank 48: 24702.74 + rank 49: 24702.89 + rank 50: 24702.93 + rank 51: 24702.86 + rank 52: 24702.94 + rank 53: 24703.12 + rank 54: 24702.89 + rank 55: 24702.86 + rank 56: 24702.68 + rank 57: 24702.70 + rank 58: 24702.70 + rank 59: 24702.71 + rank 60: 24702.71 + rank 61: 24702.72 + rank 62: 24702.68 + rank 63: 24702.67 + [2026-10-15 18:31:22] iteration 1601/ 2384 | consumed samples: 51232 | elapsed time per iteration (ms): 219676.3 | throughput per GPU (TFLOP/s/GPU): 24.3 | MFU 7.79% | FLOPs estimator: workload-aware | logical tokens/s: 76372.3 | valid tokens/s: 76372.3 | physical tokens/s: 78042.1 | loss tokens/s: 58876.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.436396E-01 | lm-global-supervised-tokens: 1.293377E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.654293E-02 | lm-token-buffer-mib: 2.348040E+02 | global_load_balancing_loss: 1.000539E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11165.005 | params rms: 0.060455 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:46:46.546859 | finish at 2026-10-17 18:18:09 + [2026-10-15 18:35:10] iteration 1602/ 2384 | consumed samples: 51264 | elapsed time per iteration (ms): 227715.4 | throughput per GPU (TFLOP/s/GPU): 19.2 | MFU 6.17% | FLOPs estimator: workload-aware | logical tokens/s: 73676.2 | valid tokens/s: 73676.2 | physical tokens/s: 75650.9 | loss tokens/s: 57087.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.221460E-01 | lm-global-supervised-tokens: 1.299977E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.617974E-02 | lm-token-buffer-mib: 2.372374E+02 | global_load_balancing_loss: 1.001463E+00 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11164.978 | params rms: 0.060455 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:27:53.434954 | finish at 2026-10-17 20:03:03 + [2026-10-15 18:38:52] iteration 1603/ 2384 | consumed samples: 51296 | elapsed time per iteration (ms): 222028.5 | throughput per GPU (TFLOP/s/GPU): 26.4 | MFU 8.45% | FLOPs estimator: workload-aware | logical tokens/s: 75563.2 | valid tokens/s: 75563.2 | physical tokens/s: 76015.9 | loss tokens/s: 59695.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.588183E-01 | lm-global-supervised-tokens: 1.325403E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.704705E-02 | lm-token-buffer-mib: 2.336117E+02 | global_load_balancing_loss: 1.000500E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11164.951 | params rms: 0.060455 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:10:04.274762 | finish at 2026-10-17 18:48:56 + [2026-10-15 18:42:35] iteration 1604/ 2384 | consumed samples: 51328 | elapsed time per iteration (ms): 222875.5 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.06% | FLOPs estimator: workload-aware | logical tokens/s: 75276.0 | valid tokens/s: 75276.0 | physical tokens/s: 75714.1 | loss tokens/s: 51591.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.082049E-01 | lm-global-supervised-tokens: 1.149857E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.691614E-02 | lm-token-buffer-mib: 2.334204E+02 | global_load_balancing_loss: 1.000977E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11164.924 | params rms: 0.060454 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:17:22.918797 | finish at 2026-10-17 18:59:58 + [2026-10-15 18:46:15] iteration 1605/ 2384 | consumed samples: 51360 | elapsed time per iteration (ms): 220043.2 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.73% | FLOPs estimator: workload-aware | logical tokens/s: 76244.8 | valid tokens/s: 76244.8 | physical tokens/s: 77128.1 | loss tokens/s: 57989.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.055983E-01 | lm-global-supervised-tokens: 1.276020E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.299143E-02 | lm-token-buffer-mib: 2.344561E+02 | global_load_balancing_loss: 1.000522E+00 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11164.900 | params rms: 0.060454 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:36:53.667114 | finish at 2026-10-17 18:23:08 + [2026-10-15 18:49:59] iteration 1606/ 2384 | consumed samples: 51392 | elapsed time per iteration (ms): 224758.7 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.80% | FLOPs estimator: workload-aware | logical tokens/s: 74645.3 | valid tokens/s: 74645.3 | physical tokens/s: 75279.0 | loss tokens/s: 59488.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.910030E-01 | lm-global-supervised-tokens: 1.337058E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.895346E-02 | lm-token-buffer-mib: 2.349564E+02 | global_load_balancing_loss: 9.997323E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11164.876 | params rms: 0.060454 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:34:22.255162 | finish at 2026-10-17 19:24:22 + [2026-10-15 18:53:41] iteration 1607/ 2384 | consumed samples: 51424 | elapsed time per iteration (ms): 221156.4 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.60% | FLOPs estimator: workload-aware | logical tokens/s: 75861.1 | valid tokens/s: 75861.1 | physical tokens/s: 76273.4 | loss tokens/s: 54085.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.852236E-01 | lm-global-supervised-tokens: 1.196130E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.372855E-02 | lm-token-buffer-mib: 2.331053E+02 | global_load_balancing_loss: 9.998047E-01 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11164.852 | params rms: 0.060454 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:43:58.528145 | finish at 2026-10-17 18:37:39 + [2026-10-15 18:57:23] iteration 1608/ 2384 | consumed samples: 51456 | elapsed time per iteration (ms): 222218.3 | throughput per GPU (TFLOP/s/GPU): 18.5 | MFU 5.93% | FLOPs estimator: workload-aware | logical tokens/s: 75498.8 | valid tokens/s: 75498.8 | physical tokens/s: 76417.5 | loss tokens/s: 58428.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.240119E-01 | lm-global-supervised-tokens: 1.298384E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.256941E-02 | lm-token-buffer-mib: 2.360428E+02 | global_load_balancing_loss: 1.000301E+00 | loss scale: 1.0 | grad norm: 0.151 | params norm: 11164.828 | params rms: 0.060454 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:54:01.396441 | finish at 2026-10-17 18:51:24 + [2026-10-15 19:01:04] iteration 1609/ 2384 | consumed samples: 51488 | elapsed time per iteration (ms): 221331.0 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.08% | FLOPs estimator: workload-aware | logical tokens/s: 75801.1 | valid tokens/s: 75801.1 | physical tokens/s: 76564.9 | loss tokens/s: 56171.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.217862E-01 | lm-global-supervised-tokens: 1.243251E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.538210E-02 | lm-token-buffer-mib: 2.339836E+02 | global_load_balancing_loss: 1.000048E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11164.801 | params rms: 0.060454 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:38:51.513798 | finish at 2026-10-17 18:39:56 + [2026-10-15 19:05:10] iteration 1610/ 2384 | consumed samples: 51520 | elapsed time per iteration (ms): 246273.9 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.54% | FLOPs estimator: workload-aware | logical tokens/s: 68124.2 | valid tokens/s: 68124.2 | physical tokens/s: 68709.2 | loss tokens/s: 49521.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.831284E-01 | lm-global-supervised-tokens: 1.219578E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.397607E-02 | lm-token-buffer-mib: 2.344455E+02 | global_load_balancing_loss: 1.000113E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11164.774 | params rms: 0.060454 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:56:55.991797 | finish at 2026-10-18 00:02:06 + [2026-10-15 19:08:54] iteration 1611/ 2384 | consumed samples: 51552 | elapsed time per iteration (ms): 223140.4 | throughput per GPU (TFLOP/s/GPU): 26.9 | MFU 8.62% | FLOPs estimator: workload-aware | logical tokens/s: 75186.7 | valid tokens/s: 75186.7 | physical tokens/s: 75779.4 | loss tokens/s: 52262.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.272648E-01 | lm-global-supervised-tokens: 1.166178E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.267123E-02 | lm-token-buffer-mib: 2.337256E+02 | global_load_balancing_loss: 1.000676E+00 | loss scale: 1.0 | grad norm: 0.173 | params norm: 11164.748 | params rms: 0.060453 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:54:47.533940 | finish at 2026-10-17 19:03:41 + [2026-10-15 19:12:38] iteration 1612/ 2384 | consumed samples: 51584 | elapsed time per iteration (ms): 223972.2 | throughput per GPU (TFLOP/s/GPU): 25.2 | MFU 8.08% | FLOPs estimator: workload-aware | logical tokens/s: 74907.3 | valid tokens/s: 74907.3 | physical tokens/s: 75943.1 | loss tokens/s: 57318.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.755521E-01 | lm-global-supervised-tokens: 1.283783E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.704232E-02 | lm-token-buffer-mib: 2.336143E+02 | global_load_balancing_loss: 9.999734E-01 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11164.723 | params rms: 0.060453 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:01:46.548091 | finish at 2026-10-17 19:14:24 + [2026-10-15 19:16:20] iteration 1613/ 2384 | consumed samples: 51616 | elapsed time per iteration (ms): 222795.8 | throughput per GPU (TFLOP/s/GPU): 24.2 | MFU 7.77% | FLOPs estimator: workload-aware | logical tokens/s: 75303.0 | valid tokens/s: 75303.0 | physical tokens/s: 75762.4 | loss tokens/s: 58968.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.742039E-01 | lm-global-supervised-tokens: 1.313797E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.417275E-02 | lm-token-buffer-mib: 2.323769E+02 | global_load_balancing_loss: 1.001451E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11164.701 | params rms: 0.060453 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:42:55.577954 | finish at 2026-10-17 18:59:16 + [2026-10-15 19:20:03] iteration 1614/ 2384 | consumed samples: 51648 | elapsed time per iteration (ms): 222273.0 | throughput per GPU (TFLOP/s/GPU): 23.8 | MFU 7.62% | FLOPs estimator: workload-aware | logical tokens/s: 75480.2 | valid tokens/s: 75480.2 | physical tokens/s: 76158.9 | loss tokens/s: 58347.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.452689E-01 | lm-global-supervised-tokens: 1.296898E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.265784E-02 | lm-token-buffer-mib: 2.341486E+02 | global_load_balancing_loss: 1.000204E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11164.677 | params rms: 0.060453 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:32:30.181363 | finish at 2026-10-17 18:52:33 + [2026-10-15 19:23:45] iteration 1615/ 2384 | consumed samples: 51680 | elapsed time per iteration (ms): 222367.3 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.14% | FLOPs estimator: workload-aware | logical tokens/s: 75448.1 | valid tokens/s: 75448.1 | physical tokens/s: 75965.4 | loss tokens/s: 54989.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.251539E-01 | lm-global-supervised-tokens: 1.222792E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.508265E-02 | lm-token-buffer-mib: 2.332177E+02 | global_load_balancing_loss: 1.000297E+00 | loss scale: 1.0 | grad norm: 0.248 | params norm: 11164.653 | params rms: 0.060453 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:30:00.483978 | finish at 2026-10-17 18:53:45 + [2026-10-15 19:27:29] iteration 1616/ 2384 | consumed samples: 51712 | elapsed time per iteration (ms): 223590.8 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.76% | FLOPs estimator: workload-aware | logical tokens/s: 75035.3 | valid tokens/s: 75035.3 | physical tokens/s: 76256.7 | loss tokens/s: 60832.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.132140E-01 | lm-global-supervised-tokens: 1.360168E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.756614E-02 | lm-token-buffer-mib: 2.333828E+02 | global_load_balancing_loss: 9.999304E-01 | loss scale: 1.0 | grad norm: 0.176 | params norm: 11164.630 | params rms: 0.060453 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:41:57.767395 | finish at 2026-10-17 19:09:26 + [2026-10-15 19:31:12] iteration 1617/ 2384 | consumed samples: 51744 | elapsed time per iteration (ms): 223752.2 | throughput per GPU (TFLOP/s/GPU): 19.6 | MFU 6.28% | FLOPs estimator: workload-aware | logical tokens/s: 74981.2 | valid tokens/s: 74981.2 | physical tokens/s: 75990.3 | loss tokens/s: 58365.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.643251E-01 | lm-global-supervised-tokens: 1.305933E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.253226E-02 | lm-token-buffer-mib: 2.356633E+02 | global_load_balancing_loss: 1.000169E+00 | loss scale: 1.0 | grad norm: 0.355 | params norm: 11164.609 | params rms: 0.060453 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:40:17.968402 | finish at 2026-10-17 19:11:30 + [2026-10-15 19:34:59] iteration 1618/ 2384 | consumed samples: 51776 | elapsed time per iteration (ms): 226752.2 | throughput per GPU (TFLOP/s/GPU): 23.8 | MFU 7.64% | FLOPs estimator: workload-aware | logical tokens/s: 73989.1 | valid tokens/s: 73989.1 | physical tokens/s: 75273.1 | loss tokens/s: 53657.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.770237E-01 | lm-global-supervised-tokens: 1.216693E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.742201E-02 | lm-token-buffer-mib: 2.335711E+02 | global_load_balancing_loss: 9.997028E-01 | loss scale: 1.0 | grad norm: 0.193 | params norm: 11164.588 | params rms: 0.060453 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:14:52.222736 | finish at 2026-10-17 19:49:51 + [2026-10-15 19:38:42] iteration 1619/ 2384 | consumed samples: 51808 | elapsed time per iteration (ms): 222865.0 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.29% | FLOPs estimator: workload-aware | logical tokens/s: 75279.5 | valid tokens/s: 75279.5 | physical tokens/s: 75823.3 | loss tokens/s: 58741.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.034857E-01 | lm-global-supervised-tokens: 1.309149E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.035521E-02 | lm-token-buffer-mib: 2.331957E+02 | global_load_balancing_loss: 1.000081E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11164.564 | params rms: 0.060452 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:21:31.708592 | finish at 2026-10-17 19:00:14 + [2026-10-15 19:42:46] iteration 1620/ 2384 | consumed samples: 51840 | elapsed time per iteration (ms): 244269.7 | throughput per GPU (TFLOP/s/GPU): 19.2 | MFU 6.14% | FLOPs estimator: workload-aware | logical tokens/s: 68683.1 | valid tokens/s: 68683.1 | physical tokens/s: 69577.4 | loss tokens/s: 52676.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.475478E-01 | lm-global-supervised-tokens: 1.286736E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.315632E-02 | lm-token-buffer-mib: 2.346102E+02 | global_load_balancing_loss: 1.000689E+00 | loss scale: 1.0 | grad norm: 0.186 | params norm: 11164.538 | params rms: 0.060452 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 3:50:22.071968 | finish at 2026-10-17 23:33:08 + [2026-10-15 19:46:30] iteration 1621/ 2384 | consumed samples: 51872 | elapsed time per iteration (ms): 223967.1 | throughput per GPU (TFLOP/s/GPU): 23.5 | MFU 7.55% | FLOPs estimator: workload-aware | logical tokens/s: 74909.1 | valid tokens/s: 74909.1 | physical tokens/s: 75686.0 | loss tokens/s: 56874.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.573409E-01 | lm-global-supervised-tokens: 1.273810E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.415423E-02 | lm-token-buffer-mib: 2.365386E+02 | global_load_balancing_loss: 1.001145E+00 | loss scale: 1.0 | grad norm: 0.313 | params norm: 11164.513 | params rms: 0.060452 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:28:06.884130 | finish at 2026-10-17 19:14:37 + [2026-10-15 19:50:13] iteration 1622/ 2384 | consumed samples: 51904 | elapsed time per iteration (ms): 222706.7 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.05% | FLOPs estimator: workload-aware | logical tokens/s: 75332.9 | valid tokens/s: 75332.9 | physical tokens/s: 76452.0 | loss tokens/s: 55826.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.521492E-01 | lm-global-supervised-tokens: 1.243284E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.819354E-02 | lm-token-buffer-mib: 2.328332E+02 | global_load_balancing_loss: 1.000426E+00 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11164.484 | params rms: 0.060452 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:08:22.522180 | finish at 2026-10-17 18:58:35 + [2026-10-15 19:53:53] iteration 1623/ 2384 | consumed samples: 51936 | elapsed time per iteration (ms): 220125.5 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.01% | FLOPs estimator: workload-aware | logical tokens/s: 76216.4 | valid tokens/s: 76216.4 | physical tokens/s: 76886.2 | loss tokens/s: 55147.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.287605E-01 | lm-global-supervised-tokens: 1.213938E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.359091E-02 | lm-token-buffer-mib: 2.334062E+02 | global_load_balancing_loss: 1.000485E+00 | loss scale: 1.0 | grad norm: 0.175 | params norm: 11164.456 | params rms: 0.060452 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:31:55.507831 | finish at 2026-10-17 18:25:49 + [2026-10-15 19:57:35] iteration 1624/ 2384 | consumed samples: 51968 | elapsed time per iteration (ms): 222398.0 | throughput per GPU (TFLOP/s/GPU): 24.5 | MFU 7.84% | FLOPs estimator: workload-aware | logical tokens/s: 75437.8 | valid tokens/s: 75437.8 | physical tokens/s: 75956.7 | loss tokens/s: 56943.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.193594E-01 | lm-global-supervised-tokens: 1.266406E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.493498E-02 | lm-token-buffer-mib: 2.338419E+02 | global_load_balancing_loss: 1.000257E+00 | loss scale: 1.0 | grad norm: 0.128 | params norm: 11164.429 | params rms: 0.060452 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:57:02.470942 | finish at 2026-10-17 18:54:38 + [2026-10-15 20:01:17] iteration 1625/ 2384 | consumed samples: 52000 | elapsed time per iteration (ms): 221737.4 | throughput per GPU (TFLOP/s/GPU): 24.1 | MFU 7.73% | FLOPs estimator: workload-aware | logical tokens/s: 75662.3 | valid tokens/s: 75662.3 | physical tokens/s: 76023.6 | loss tokens/s: 61789.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.534713E-01 | lm-global-supervised-tokens: 1.370106E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.540727E-02 | lm-token-buffer-mib: 2.332488E+02 | global_load_balancing_loss: 1.000670E+00 | loss scale: 1.0 | grad norm: 0.125 | params norm: 11164.401 | params rms: 0.060452 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:44:58.719576 | finish at 2026-10-17 18:46:16 + [2026-10-15 20:05:04] iteration 1626/ 2384 | consumed samples: 52032 | elapsed time per iteration (ms): 227061.4 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.94% | FLOPs estimator: workload-aware | logical tokens/s: 73888.2 | valid tokens/s: 73888.2 | physical tokens/s: 74865.4 | loss tokens/s: 53033.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.752139E-01 | lm-global-supervised-tokens: 1.204189E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.164640E-02 | lm-token-buffer-mib: 2.377477E+02 | global_load_balancing_loss: 9.992545E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11164.369 | params rms: 0.060451 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:48:32.565730 | finish at 2026-10-17 19:53:37 + [2026-10-15 20:08:50] iteration 1627/ 2384 | consumed samples: 52064 | elapsed time per iteration (ms): 226299.6 | throughput per GPU (TFLOP/s/GPU): 25.1 | MFU 8.05% | FLOPs estimator: workload-aware | logical tokens/s: 74137.1 | valid tokens/s: 74137.0 | physical tokens/s: 74558.9 | loss tokens/s: 58161.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.118980E-01 | lm-global-supervised-tokens: 1.316197E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.602324E-02 | lm-token-buffer-mib: 2.339877E+02 | global_load_balancing_loss: 1.000084E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11164.335 | params rms: 0.060451 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:35:08.814440 | finish at 2026-10-17 19:43:59 + [2026-10-15 20:12:37] iteration 1628/ 2384 | consumed samples: 52096 | elapsed time per iteration (ms): 226806.5 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.37% | FLOPs estimator: workload-aware | logical tokens/s: 73971.4 | valid tokens/s: 73971.4 | physical tokens/s: 75127.0 | loss tokens/s: 54427.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.027712E-01 | lm-global-supervised-tokens: 1.234461E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.667064E-02 | lm-token-buffer-mib: 2.382021E+02 | global_load_balancing_loss: 1.001033E+00 | loss scale: 1.0 | grad norm: 0.147 | params norm: 11164.301 | params rms: 0.060451 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:37:45.709462 | finish at 2026-10-17 19:50:23 + [2026-10-15 20:16:18] iteration 1629/ 2384 | consumed samples: 52128 | elapsed time per iteration (ms): 221072.7 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.53% | FLOPs estimator: workload-aware | logical tokens/s: 75890.0 | valid tokens/s: 75890.0 | physical tokens/s: 76653.2 | loss tokens/s: 56869.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.402995E-01 | lm-global-supervised-tokens: 1.257219E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.256240E-02 | lm-token-buffer-mib: 2.340138E+02 | global_load_balancing_loss: 1.000930E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11164.268 | params rms: 0.060451 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:21:49.889958 | finish at 2026-10-17 18:38:08 + [2026-10-15 20:20:28] iteration 1630/ 2384 | consumed samples: 52160 | elapsed time per iteration (ms): 249805.2 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.69% | FLOPs estimator: workload-aware | logical tokens/s: 67161.0 | valid tokens/s: 67161.0 | physical tokens/s: 68002.0 | loss tokens/s: 49333.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.131220E-01 | lm-global-supervised-tokens: 1.232381E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.528556E-02 | lm-token-buffer-mib: 2.355140E+02 | global_load_balancing_loss: 9.996846E-01 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11164.236 | params rms: 0.060451 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:19:13.142807 | finish at 2026-10-18 00:39:41 + [2026-10-15 20:24:10] iteration 1631/ 2384 | consumed samples: 52192 | elapsed time per iteration (ms): 221844.4 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.75% | FLOPs estimator: workload-aware | logical tokens/s: 75626.0 | valid tokens/s: 75626.0 | physical tokens/s: 76383.0 | loss tokens/s: 58845.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.744581E-01 | lm-global-supervised-tokens: 1.305456E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.738232E-02 | lm-token-buffer-mib: 2.336329E+02 | global_load_balancing_loss: 9.994727E-01 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11164.203 | params rms: 0.060451 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:24:08.849663 | finish at 2026-10-17 18:48:19 + [2026-10-15 20:27:55] iteration 1632/ 2384 | consumed samples: 52224 | elapsed time per iteration (ms): 225073.3 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.43% | FLOPs estimator: workload-aware | logical tokens/s: 74541.0 | valid tokens/s: 74541.0 | physical tokens/s: 75818.7 | loss tokens/s: 56859.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.263987E-01 | lm-global-supervised-tokens: 1.279758E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.189999E-02 | lm-token-buffer-mib: 2.359135E+02 | global_load_balancing_loss: 1.000342E+00 | loss scale: 1.0 | grad norm: 0.153 | params norm: 11164.167 | params rms: 0.060450 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:00:55.156834 | finish at 2026-10-17 19:28:50 + [2026-10-15 20:31:37] iteration 1633/ 2384 | consumed samples: 52256 | elapsed time per iteration (ms): 221495.8 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.45% | FLOPs estimator: workload-aware | logical tokens/s: 75745.0 | valid tokens/s: 75745.0 | physical tokens/s: 76310.2 | loss tokens/s: 55444.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.667064E-01 | lm-global-supervised-tokens: 1.228074E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.644848E-02 | lm-token-buffer-mib: 2.342635E+02 | global_load_balancing_loss: 1.001048E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11164.135 | params rms: 0.060450 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:12:23.376074 | finish at 2026-10-17 18:44:00 + [2026-10-15 20:35:20] iteration 1634/ 2384 | consumed samples: 52288 | elapsed time per iteration (ms): 223766.9 | throughput per GPU (TFLOP/s/GPU): 23.4 | MFU 7.49% | FLOPs estimator: workload-aware | logical tokens/s: 74976.1 | valid tokens/s: 74976.1 | physical tokens/s: 75846.6 | loss tokens/s: 55784.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.253669E-01 | lm-global-supervised-tokens: 1.248266E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.713137E-02 | lm-token-buffer-mib: 2.349532E+02 | global_load_balancing_loss: 1.000606E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11164.102 | params rms: 0.060450 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:37:05.189531 | finish at 2026-10-17 19:12:26 + [2026-10-15 20:39:03] iteration 1635/ 2384 | consumed samples: 52320 | elapsed time per iteration (ms): 222427.7 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.20% | FLOPs estimator: workload-aware | logical tokens/s: 75427.6 | valid tokens/s: 75427.6 | physical tokens/s: 76568.6 | loss tokens/s: 59041.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.859859E-01 | lm-global-supervised-tokens: 1.313248E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.504752E-02 | lm-token-buffer-mib: 2.356234E+02 | global_load_balancing_loss: 9.993528E-01 | loss scale: 1.0 | grad norm: 0.197 | params norm: 11164.068 | params rms: 0.060450 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:16:38.330546 | finish at 2026-10-17 18:55:41 + [2026-10-15 20:42:46] iteration 1636/ 2384 | consumed samples: 52352 | elapsed time per iteration (ms): 223693.4 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.96% | FLOPs estimator: workload-aware | logical tokens/s: 75000.8 | valid tokens/s: 75000.8 | physical tokens/s: 75826.6 | loss tokens/s: 54828.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.465418E-01 | lm-global-supervised-tokens: 1.226482E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.323633E-02 | lm-token-buffer-mib: 2.350280E+02 | global_load_balancing_loss: 1.001405E+00 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11164.036 | params rms: 0.060450 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:28:42.667588 | finish at 2026-10-17 19:11:29 + [2026-10-15 20:46:27] iteration 1637/ 2384 | consumed samples: 52384 | elapsed time per iteration (ms): 220479.2 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.67% | FLOPs estimator: workload-aware | logical tokens/s: 76094.2 | valid tokens/s: 76094.2 | physical tokens/s: 76760.2 | loss tokens/s: 53230.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.969100E-01 | lm-global-supervised-tokens: 1.173620E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.731265E-02 | lm-token-buffer-mib: 2.352816E+02 | global_load_balancing_loss: 1.000166E+00 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11164.006 | params rms: 0.060449 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:44:57.953944 | finish at 2026-10-17 18:31:25 + [2026-10-15 20:50:09] iteration 1638/ 2384 | consumed samples: 52416 | elapsed time per iteration (ms): 222241.3 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.98% | FLOPs estimator: workload-aware | logical tokens/s: 75490.9 | valid tokens/s: 75490.9 | physical tokens/s: 76006.8 | loss tokens/s: 54574.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.449643E-01 | lm-global-supervised-tokens: 1.212876E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.130559E-02 | lm-token-buffer-mib: 2.333349E+02 | global_load_balancing_loss: 9.997108E-01 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11163.973 | params rms: 0.060449 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:03:11.981955 | finish at 2026-10-17 18:53:21 + [2026-10-15 20:53:51] iteration 1639/ 2384 | consumed samples: 52448 | elapsed time per iteration (ms): 222281.2 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.42% | FLOPs estimator: workload-aware | logical tokens/s: 75477.3 | valid tokens/s: 75477.3 | physical tokens/s: 75866.4 | loss tokens/s: 56573.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.235597E-01 | lm-global-supervised-tokens: 1.257517E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.310549E-02 | lm-token-buffer-mib: 2.333214E+02 | global_load_balancing_loss: 9.999558E-01 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11163.938 | params rms: 0.060449 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:59:59.473523 | finish at 2026-10-17 18:53:51 + [2026-10-15 20:57:56] iteration 1640/ 2384 | consumed samples: 52480 | elapsed time per iteration (ms): 244139.6 | throughput per GPU (TFLOP/s/GPU): 18.8 | MFU 6.04% | FLOPs estimator: workload-aware | logical tokens/s: 68719.7 | valid tokens/s: 68719.7 | physical tokens/s: 69588.5 | loss tokens/s: 53473.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.972888E-01 | lm-global-supervised-tokens: 1.305498E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.546743E-02 | lm-token-buffer-mib: 2.337522E+02 | global_load_balancing_loss: 1.001145E+00 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11163.903 | params rms: 0.060449 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:27:19.865799 | finish at 2026-10-17 23:25:15 +times across ranks (ms): + save-checkpoint: + rank 0: 24609.32 + rank 1: 24609.32 + rank 2: 24609.34 + rank 3: 24609.35 + rank 4: 24609.33 + rank 5: 24609.35 + rank 6: 24609.36 + rank 7: 24609.33 + rank 8: 24608.96 + rank 9: 24608.98 + rank 10: 24608.96 + rank 11: 24608.95 + rank 12: 24608.94 + rank 13: 24608.93 + rank 14: 24609.00 + rank 15: 24608.94 + rank 16: 24608.80 + rank 17: 24608.78 + rank 18: 24608.82 + rank 19: 24608.81 + rank 20: 24608.83 + rank 21: 24608.80 + rank 22: 24608.81 + rank 23: 24608.79 + rank 24: 24608.95 + rank 25: 24608.95 + rank 26: 24608.95 + rank 27: 24608.96 + rank 28: 24608.94 + rank 29: 24608.98 + rank 30: 24608.95 + rank 31: 24608.97 + rank 32: 24608.62 + rank 33: 24608.68 + rank 34: 24608.62 + rank 35: 24608.64 + rank 36: 24608.66 + rank 37: 24608.63 + rank 38: 24608.54 + rank 39: 24608.54 + rank 40: 24608.69 + rank 41: 24608.65 + rank 42: 24608.64 + rank 43: 24608.65 + rank 44: 24608.67 + rank 45: 24608.65 + rank 46: 24608.62 + rank 47: 24608.62 + rank 48: 24609.13 + rank 49: 24609.14 + rank 50: 24609.16 + rank 51: 24609.15 + rank 52: 24609.14 + rank 53: 24609.16 + rank 54: 24609.13 + rank 55: 24609.12 + rank 56: 24608.79 + rank 57: 24608.82 + rank 58: 24608.82 + rank 59: 24608.80 + rank 60: 24608.83 + rank 61: 24608.79 + rank 62: 24608.79 + rank 63: 24608.79 + [2026-10-15 21:02:07] iteration 1641/ 2384 | consumed samples: 52512 | elapsed time per iteration (ms): 226917.2 | throughput per GPU (TFLOP/s/GPU): 23.9 | MFU 7.67% | FLOPs estimator: workload-aware | logical tokens/s: 73935.2 | valid tokens/s: 73935.2 | physical tokens/s: 74379.1 | loss tokens/s: 57076.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.592227E-01 | lm-global-supervised-tokens: 1.295151E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.114378E-01 | lm-token-buffer-mib: 2.335431E+02 | global_load_balancing_loss: 1.001195E+00 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11163.868 | params rms: 0.060449 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:49:59.452385 | finish at 2026-10-17 19:52:07 + [2026-10-15 21:05:55] iteration 1642/ 2384 | consumed samples: 52544 | elapsed time per iteration (ms): 227425.7 | throughput per GPU (TFLOP/s/GPU): 19.1 | MFU 6.13% | FLOPs estimator: workload-aware | logical tokens/s: 73770.1 | valid tokens/s: 73770.1 | physical tokens/s: 74378.8 | loss tokens/s: 57590.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.069745E-01 | lm-global-supervised-tokens: 1.309764E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.717461E-02 | lm-token-buffer-mib: 2.335736E+02 | global_load_balancing_loss: 1.001042E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11163.835 | params rms: 0.060449 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:52:29.902090 | finish at 2026-10-17 19:58:25 + [2026-10-15 21:09:39] iteration 1643/ 2384 | consumed samples: 52576 | elapsed time per iteration (ms): 223974.0 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.15% | FLOPs estimator: workload-aware | logical tokens/s: 74906.8 | valid tokens/s: 74906.8 | physical tokens/s: 75589.1 | loss tokens/s: 58895.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.060059E-01 | lm-global-supervised-tokens: 1.319100E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.732527E-02 | lm-token-buffer-mib: 2.339486E+02 | global_load_balancing_loss: 1.000361E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11163.807 | params rms: 0.060448 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:06:04.703773 | finish at 2026-10-17 19:15:43 + [2026-10-15 21:13:21] iteration 1644/ 2384 | consumed samples: 52608 | elapsed time per iteration (ms): 221885.6 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.98% | FLOPs estimator: workload-aware | logical tokens/s: 75611.9 | valid tokens/s: 75611.9 | physical tokens/s: 76210.2 | loss tokens/s: 53942.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.761885E-01 | lm-global-supervised-tokens: 1.196912E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.257338E-02 | lm-token-buffer-mib: 2.348543E+02 | global_load_balancing_loss: 1.000720E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11163.780 | params rms: 0.060448 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:36:35.308604 | finish at 2026-10-17 18:49:56 + [2026-10-15 21:17:01] iteration 1645/ 2384 | consumed samples: 52640 | elapsed time per iteration (ms): 219975.6 | throughput per GPU (TFLOP/s/GPU): 24.4 | MFU 7.82% | FLOPs estimator: workload-aware | logical tokens/s: 76268.5 | valid tokens/s: 76268.5 | physical tokens/s: 76775.1 | loss tokens/s: 61475.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.983437E-01 | lm-global-supervised-tokens: 1.352300E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.270705E-02 | lm-token-buffer-mib: 2.333116E+02 | global_load_balancing_loss: 1.000165E+00 | loss scale: 1.0 | grad norm: 0.151 | params norm: 11163.749 | params rms: 0.060448 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:09:21.937393 | finish at 2026-10-17 18:26:23 + [2026-10-15 21:20:41] iteration 1646/ 2384 | consumed samples: 52672 | elapsed time per iteration (ms): 220493.3 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.96% | FLOPs estimator: workload-aware | logical tokens/s: 76089.4 | valid tokens/s: 76089.4 | physical tokens/s: 76491.5 | loss tokens/s: 54369.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.678120E-01 | lm-global-supervised-tokens: 1.198807E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.621988E-02 | lm-token-buffer-mib: 2.335460E+02 | global_load_balancing_loss: 9.990630E-01 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11163.713 | params rms: 0.060448 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:12:04.077189 | finish at 2026-10-17 18:32:45 + [2026-10-15 21:24:23] iteration 1647/ 2384 | consumed samples: 52704 | elapsed time per iteration (ms): 222066.4 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.81% | FLOPs estimator: workload-aware | logical tokens/s: 75550.3 | valid tokens/s: 75550.3 | physical tokens/s: 76161.4 | loss tokens/s: 58365.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.364552E-01 | lm-global-supervised-tokens: 1.296108E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.529630E-02 | lm-token-buffer-mib: 2.345125E+02 | global_load_balancing_loss: 9.986477E-01 | loss scale: 1.0 | grad norm: 0.095 | params norm: 11163.680 | params rms: 0.060448 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:27:42.924538 | finish at 2026-10-17 18:52:06 + [2026-10-15 21:28:02] iteration 1648/ 2384 | consumed samples: 52736 | elapsed time per iteration (ms): 218355.0 | throughput per GPU (TFLOP/s/GPU): 24.1 | MFU 7.74% | FLOPs estimator: workload-aware | logical tokens/s: 76834.3 | valid tokens/s: 76834.3 | physical tokens/s: 77198.6 | loss tokens/s: 56826.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.109925E-01 | lm-global-supervised-tokens: 1.240837E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.271508E-02 | lm-token-buffer-mib: 2.324753E+02 | global_load_balancing_loss: 9.992774E-01 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11163.654 | params rms: 0.060448 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:38:29.314934 | finish at 2026-10-17 18:06:31 + [2026-10-15 21:31:44] iteration 1649/ 2384 | consumed samples: 52768 | elapsed time per iteration (ms): 222749.8 | throughput per GPU (TFLOP/s/GPU): 23.9 | MFU 7.66% | FLOPs estimator: workload-aware | logical tokens/s: 75318.6 | valid tokens/s: 75318.6 | physical tokens/s: 76120.0 | loss tokens/s: 57814.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.744648E-01 | lm-global-supervised-tokens: 1.287815E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.765582E-02 | lm-token-buffer-mib: 2.351690E+02 | global_load_balancing_loss: 1.001276E+00 | loss scale: 1.0 | grad norm: 0.116 | params norm: 11163.623 | params rms: 0.060447 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:28:41.072484 | finish at 2026-10-17 19:00:25 + [2026-10-15 21:35:45] iteration 1650/ 2384 | consumed samples: 52800 | elapsed time per iteration (ms): 241191.0 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.29% | FLOPs estimator: workload-aware | logical tokens/s: 69559.8 | valid tokens/s: 69559.8 | physical tokens/s: 70038.0 | loss tokens/s: 55244.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.918230E-01 | lm-global-supervised-tokens: 1.332436E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.302126E-02 | lm-token-buffer-mib: 2.325098E+02 | global_load_balancing_loss: 1.000314E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11163.596 | params rms: 0.060447 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:10:34.171939 | finish at 2026-10-17 22:46:20 + [2026-10-15 21:39:28] iteration 1651/ 2384 | consumed samples: 52832 | elapsed time per iteration (ms): 222026.7 | throughput per GPU (TFLOP/s/GPU): 27.1 | MFU 8.69% | FLOPs estimator: workload-aware | logical tokens/s: 75563.8 | valid tokens/s: 75563.8 | physical tokens/s: 76264.1 | loss tokens/s: 55473.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.150899E-01 | lm-global-supervised-tokens: 1.231665E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.377533E-02 | lm-token-buffer-mib: 2.340492E+02 | global_load_balancing_loss: 1.001128E+00 | loss scale: 1.0 | grad norm: 0.142 | params norm: 11163.568 | params rms: 0.060447 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:12:25.599775 | finish at 2026-10-17 18:51:53 + [2026-10-15 21:43:09] iteration 1652/ 2384 | consumed samples: 52864 | elapsed time per iteration (ms): 221963.7 | throughput per GPU (TFLOP/s/GPU): 18.8 | MFU 6.03% | FLOPs estimator: workload-aware | logical tokens/s: 75585.2 | valid tokens/s: 75585.2 | physical tokens/s: 76237.9 | loss tokens/s: 57513.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.706677E-01 | lm-global-supervised-tokens: 1.276588E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.199882E-02 | lm-token-buffer-mib: 2.336302E+02 | global_load_balancing_loss: 1.001182E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11163.540 | params rms: 0.060447 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:07:57.436469 | finish at 2026-10-17 18:51:07 + [2026-10-15 21:46:54] iteration 1653/ 2384 | consumed samples: 52896 | elapsed time per iteration (ms): 224248.3 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.42% | FLOPs estimator: workload-aware | logical tokens/s: 74815.3 | valid tokens/s: 74815.3 | physical tokens/s: 75762.3 | loss tokens/s: 57156.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.831245E-01 | lm-global-supervised-tokens: 1.281718E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.430407E-02 | lm-token-buffer-mib: 2.364626E+02 | global_load_balancing_loss: 1.002122E+00 | loss scale: 1.0 | grad norm: 0.195 | params norm: 11163.515 | params rms: 0.060447 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:32:05.498467 | finish at 2026-10-17 19:18:59 + [2026-10-15 21:50:41] iteration 1654/ 2384 | consumed samples: 52928 | elapsed time per iteration (ms): 227621.3 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.76% | FLOPs estimator: workload-aware | logical tokens/s: 73706.6 | valid tokens/s: 73706.6 | physical tokens/s: 74749.6 | loss tokens/s: 59128.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.716003E-01 | lm-global-supervised-tokens: 1.345882E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.333896E-02 | lm-token-buffer-mib: 2.345704E+02 | global_load_balancing_loss: 1.000683E+00 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11163.492 | params rms: 0.060447 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:09:23.561170 | finish at 2026-10-17 20:00:05 + [2026-10-15 21:54:28] iteration 1655/ 2384 | consumed samples: 52960 | elapsed time per iteration (ms): 226729.9 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.94% | FLOPs estimator: workload-aware | logical tokens/s: 73996.1 | valid tokens/s: 73996.1 | physical tokens/s: 74961.0 | loss tokens/s: 56660.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.874749E-01 | lm-global-supervised-tokens: 1.284656E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.803181E-02 | lm-token-buffer-mib: 2.352471E+02 | global_load_balancing_loss: 9.998909E-01 | loss scale: 1.0 | grad norm: 0.125 | params norm: 11163.470 | params rms: 0.060447 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:54:46.125693 | finish at 2026-10-17 19:49:14 + [2026-10-15 21:58:13] iteration 1656/ 2384 | consumed samples: 52992 | elapsed time per iteration (ms): 225227.3 | throughput per GPU (TFLOP/s/GPU): 25.0 | MFU 8.02% | FLOPs estimator: workload-aware | logical tokens/s: 74490.1 | valid tokens/s: 74490.1 | physical tokens/s: 75136.6 | loss tokens/s: 54330.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.820808E-01 | lm-global-supervised-tokens: 1.223675E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.295979E-02 | lm-token-buffer-mib: 2.333616E+02 | global_load_balancing_loss: 9.996611E-01 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11163.444 | params rms: 0.060446 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:32:45.439808 | finish at 2026-10-17 19:30:59 + [2026-10-15 22:01:55] iteration 1657/ 2384 | consumed samples: 53024 | elapsed time per iteration (ms): 222047.3 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.06% | FLOPs estimator: workload-aware | logical tokens/s: 75556.7 | valid tokens/s: 75556.7 | physical tokens/s: 75924.4 | loss tokens/s: 59516.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.589117E-01 | lm-global-supervised-tokens: 1.321537E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.472273E-02 | lm-token-buffer-mib: 2.322669E+02 | global_load_balancing_loss: 9.999770E-01 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11163.417 | params rms: 0.060446 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:50:28.406239 | finish at 2026-10-17 18:52:24 + [2026-10-15 22:05:38] iteration 1658/ 2384 | consumed samples: 53056 | elapsed time per iteration (ms): 222330.6 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.29% | FLOPs estimator: workload-aware | logical tokens/s: 75460.5 | valid tokens/s: 75460.5 | physical tokens/s: 76210.3 | loss tokens/s: 53600.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.279137E-01 | lm-global-supervised-tokens: 1.191695E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.679681E-02 | lm-token-buffer-mib: 2.336868E+02 | global_load_balancing_loss: 9.990127E-01 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11163.392 | params rms: 0.060446 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:50:12.005751 | finish at 2026-10-17 18:55:50 + [2026-10-15 22:09:22] iteration 1659/ 2384 | consumed samples: 53088 | elapsed time per iteration (ms): 223916.9 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.83% | FLOPs estimator: workload-aware | logical tokens/s: 74925.9 | valid tokens/s: 74925.9 | physical tokens/s: 76252.0 | loss tokens/s: 59716.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.993735E-01 | lm-global-supervised-tokens: 1.337153E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.199350E-02 | lm-token-buffer-mib: 2.359268E+02 | global_load_balancing_loss: 1.000244E+00 | loss scale: 1.0 | grad norm: 0.156 | params norm: 11163.366 | params rms: 0.060446 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:05:39.780444 | finish at 2026-10-17 19:15:01 + [2026-10-15 22:13:29] iteration 1660/ 2384 | consumed samples: 53120 | elapsed time per iteration (ms): 246912.5 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.19% | FLOPs estimator: workload-aware | logical tokens/s: 67947.9 | valid tokens/s: 67947.9 | physical tokens/s: 68848.8 | loss tokens/s: 50736.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.225377E-01 | lm-global-supervised-tokens: 1.252744E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.323498E-02 | lm-token-buffer-mib: 2.370051E+02 | global_load_balancing_loss: 9.993770E-01 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11163.339 | params rms: 0.060446 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:39:24.621449 | finish at 2026-10-17 23:52:53 + [2026-10-15 22:17:08] iteration 1661/ 2384 | consumed samples: 53152 | elapsed time per iteration (ms): 219852.6 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.30% | FLOPs estimator: workload-aware | logical tokens/s: 76311.0 | valid tokens/s: 76311.0 | physical tokens/s: 76990.0 | loss tokens/s: 58665.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.939252E-01 | lm-global-supervised-tokens: 1.289779E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.014152E-02 | lm-token-buffer-mib: 2.335233E+02 | global_load_balancing_loss: 1.000887E+00 | loss scale: 1.0 | grad norm: 0.148 | params norm: 11163.313 | params rms: 0.060446 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:09:13.446591 | finish at 2026-10-17 18:26:22 + [2026-10-15 22:20:48] iteration 1662/ 2384 | consumed samples: 53184 | elapsed time per iteration (ms): 219822.0 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.47% | FLOPs estimator: workload-aware | logical tokens/s: 76321.6 | valid tokens/s: 76321.6 | physical tokens/s: 77138.5 | loss tokens/s: 59570.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.258815E-01 | lm-global-supervised-tokens: 1.309496E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.261280E-02 | lm-token-buffer-mib: 2.336057E+02 | global_load_balancing_loss: 1.001829E+00 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11163.287 | params rms: 0.060446 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:05:11.473347 | finish at 2026-10-17 18:26:00 + [2026-10-15 22:24:29] iteration 1663/ 2384 | consumed samples: 53216 | elapsed time per iteration (ms): 221291.3 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.41% | FLOPs estimator: workload-aware | logical tokens/s: 75815.1 | valid tokens/s: 75815.1 | physical tokens/s: 76675.2 | loss tokens/s: 54057.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.545366E-01 | lm-global-supervised-tokens: 1.196240E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.420351E-02 | lm-token-buffer-mib: 2.334861E+02 | global_load_balancing_loss: 1.001632E+00 | loss scale: 1.0 | grad norm: 0.155 | params norm: 11163.261 | params rms: 0.060445 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:19:10.996916 | finish at 2026-10-17 18:43:40 + [2026-10-15 22:28:11] iteration 1664/ 2384 | consumed samples: 53248 | elapsed time per iteration (ms): 221617.4 | throughput per GPU (TFLOP/s/GPU): 23.2 | MFU 7.43% | FLOPs estimator: workload-aware | logical tokens/s: 75703.4 | valid tokens/s: 75703.4 | physical tokens/s: 76241.5 | loss tokens/s: 57058.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.868587E-01 | lm-global-supervised-tokens: 1.264524E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.930377E-02 | lm-token-buffer-mib: 2.335970E+02 | global_load_balancing_loss: 1.000430E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11163.230 | params rms: 0.060445 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:19:24.549408 | finish at 2026-10-17 18:47:36 + [2026-10-15 22:31:53] iteration 1665/ 2384 | consumed samples: 53280 | elapsed time per iteration (ms): 221954.3 | throughput per GPU (TFLOP/s/GPU): 25.8 | MFU 8.26% | FLOPs estimator: workload-aware | logical tokens/s: 75588.5 | valid tokens/s: 75588.5 | physical tokens/s: 75926.8 | loss tokens/s: 52336.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.887035E-01 | lm-global-supervised-tokens: 1.161630E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.137957E-02 | lm-token-buffer-mib: 2.329753E+02 | global_load_balancing_loss: 1.001003E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11163.199 | params rms: 0.060445 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:19:45.137533 | finish at 2026-10-17 18:51:38 + [2026-10-15 22:35:37] iteration 1666/ 2384 | consumed samples: 53312 | elapsed time per iteration (ms): 223769.1 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.97% | FLOPs estimator: workload-aware | logical tokens/s: 74975.5 | valid tokens/s: 74975.5 | physical tokens/s: 75626.5 | loss tokens/s: 57337.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.055722E-01 | lm-global-supervised-tokens: 1.283039E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.239872E-02 | lm-token-buffer-mib: 2.338348E+02 | global_load_balancing_loss: 1.000229E+00 | loss scale: 1.0 | grad norm: 0.095 | params norm: 11163.169 | params rms: 0.060445 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:37:46.207602 | finish at 2026-10-17 19:13:23 + [2026-10-15 22:39:19] iteration 1667/ 2384 | consumed samples: 53344 | elapsed time per iteration (ms): 222357.0 | throughput per GPU (TFLOP/s/GPU): 24.4 | MFU 7.82% | FLOPs estimator: workload-aware | logical tokens/s: 75451.6 | valid tokens/s: 75451.6 | physical tokens/s: 75833.7 | loss tokens/s: 57433.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.630339E-01 | lm-global-supervised-tokens: 1.277072E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.875229E-02 | lm-token-buffer-mib: 2.330319E+02 | global_load_balancing_loss: 9.976596E-01 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11163.139 | params rms: 0.060445 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:17:09.959508 | finish at 2026-10-17 18:56:29 + [2026-10-15 22:43:05] iteration 1668/ 2384 | consumed samples: 53376 | elapsed time per iteration (ms): 225485.2 | throughput per GPU (TFLOP/s/GPU): 19.3 | MFU 6.18% | FLOPs estimator: workload-aware | logical tokens/s: 74404.8 | valid tokens/s: 74404.8 | physical tokens/s: 75179.8 | loss tokens/s: 55270.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.032770E-01 | lm-global-supervised-tokens: 1.246276E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.318662E-02 | lm-token-buffer-mib: 2.353250E+02 | global_load_balancing_loss: 9.983108E-01 | loss scale: 1.0 | grad norm: 0.103 | params norm: 11163.108 | params rms: 0.060445 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:50:47.435754 | finish at 2026-10-17 19:33:52 + [2026-10-15 22:46:50] iteration 1669/ 2384 | consumed samples: 53408 | elapsed time per iteration (ms): 225402.3 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.77% | FLOPs estimator: workload-aware | logical tokens/s: 74432.2 | valid tokens/s: 74432.2 | physical tokens/s: 75187.9 | loss tokens/s: 52974.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.910423E-01 | lm-global-supervised-tokens: 1.194059E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.172472E-02 | lm-token-buffer-mib: 2.353316E+02 | global_load_balancing_loss: 1.000227E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11163.075 | params rms: 0.060444 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:46:02.638448 | finish at 2026-10-17 19:32:53 + [2026-10-15 22:51:00] iteration 1670/ 2384 | consumed samples: 53440 | elapsed time per iteration (ms): 250336.4 | throughput per GPU (TFLOP/s/GPU): 19.1 | MFU 6.11% | FLOPs estimator: workload-aware | logical tokens/s: 67018.7 | valid tokens/s: 67018.6 | physical tokens/s: 67981.7 | loss tokens/s: 50479.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.271255E-01 | lm-global-supervised-tokens: 1.263693E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.868245E-02 | lm-token-buffer-mib: 2.363080E+02 | global_load_balancing_loss: 1.001685E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11163.041 | params rms: 0.060444 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:39:00.214647 | finish at 2026-10-18 00:30:01 + [2026-10-15 22:54:43] iteration 1671/ 2384 | consumed samples: 53472 | elapsed time per iteration (ms): 223069.2 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.41% | FLOPs estimator: workload-aware | logical tokens/s: 75210.6 | valid tokens/s: 75210.6 | physical tokens/s: 75756.5 | loss tokens/s: 60787.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.364605E-01 | lm-global-supervised-tokens: 1.355980E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.208711E-02 | lm-token-buffer-mib: 2.326280E+02 | global_load_balancing_loss: 1.000807E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11163.010 | params rms: 0.060444 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:10:48.329598 | finish at 2026-10-17 19:05:32 + [2026-10-15 22:58:27] iteration 1672/ 2384 | consumed samples: 53504 | elapsed time per iteration (ms): 223366.7 | throughput per GPU (TFLOP/s/GPU): 23.2 | MFU 7.45% | FLOPs estimator: workload-aware | logical tokens/s: 75110.4 | valid tokens/s: 75110.4 | physical tokens/s: 75701.7 | loss tokens/s: 59775.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.398298E-01 | lm-global-supervised-tokens: 1.335188E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.224615E-02 | lm-token-buffer-mib: 2.340317E+02 | global_load_balancing_loss: 1.001367E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11162.978 | params rms: 0.060444 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:10:37.107498 | finish at 2026-10-17 19:09:04 + [2026-10-15 23:02:08] iteration 1673/ 2384 | consumed samples: 53536 | elapsed time per iteration (ms): 220879.9 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.48% | FLOPs estimator: workload-aware | logical tokens/s: 75956.2 | valid tokens/s: 75956.2 | physical tokens/s: 76375.8 | loss tokens/s: 60330.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.057370E-01 | lm-global-supervised-tokens: 1.332568E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 5.581454E-02 | lm-token-buffer-mib: 2.330871E+02 | global_load_balancing_loss: 1.000990E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11162.944 | params rms: 0.060444 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:37:25.617191 | finish at 2026-10-17 18:39:33 + [2026-10-15 23:05:49] iteration 1674/ 2384 | consumed samples: 53568 | elapsed time per iteration (ms): 221496.4 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.06% | FLOPs estimator: workload-aware | logical tokens/s: 75744.8 | valid tokens/s: 75744.8 | physical tokens/s: 76590.3 | loss tokens/s: 61083.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.135680E-01 | lm-global-supervised-tokens: 1.352966E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.385765E-02 | lm-token-buffer-mib: 2.337858E+02 | global_load_balancing_loss: 1.000230E+00 | loss scale: 1.0 | grad norm: 0.161 | params norm: 11162.911 | params rms: 0.060444 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:41:02.448485 | finish at 2026-10-17 18:46:52 + [2026-10-15 23:09:30] iteration 1675/ 2384 | consumed samples: 53600 | elapsed time per iteration (ms): 220732.4 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.83% | FLOPs estimator: workload-aware | logical tokens/s: 76006.9 | valid tokens/s: 76006.9 | physical tokens/s: 76444.0 | loss tokens/s: 52370.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.869927E-01 | lm-global-supervised-tokens: 1.155978E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.199517E-02 | lm-token-buffer-mib: 2.340290E+02 | global_load_balancing_loss: 1.001508E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11162.880 | params rms: 0.060443 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:28:19.287955 | finish at 2026-10-17 18:37:49 + [2026-10-15 23:13:11] iteration 1676/ 2384 | consumed samples: 53632 | elapsed time per iteration (ms): 220802.5 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.81% | FLOPs estimator: workload-aware | logical tokens/s: 75982.8 | valid tokens/s: 75982.8 | physical tokens/s: 76802.6 | loss tokens/s: 57469.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.630567E-01 | lm-global-supervised-tokens: 1.268947E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.893396E-02 | lm-token-buffer-mib: 2.355288E+02 | global_load_balancing_loss: 9.997043E-01 | loss scale: 1.0 | grad norm: 0.239 | params norm: 11162.850 | params rms: 0.060443 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:25:28.142830 | finish at 2026-10-17 18:38:39 + [2026-10-15 23:16:50] iteration 1677/ 2384 | consumed samples: 53664 | elapsed time per iteration (ms): 219290.5 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.02% | FLOPs estimator: workload-aware | logical tokens/s: 76506.7 | valid tokens/s: 76506.7 | physical tokens/s: 77265.6 | loss tokens/s: 58495.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.160413E-01 | lm-global-supervised-tokens: 1.282744E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.752626E-02 | lm-token-buffer-mib: 2.341983E+02 | global_load_balancing_loss: 9.994338E-01 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11162.817 | params rms: 0.060443 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:03:58.370637 | finish at 2026-10-17 18:20:48 + [2026-10-15 23:20:33] iteration 1678/ 2384 | consumed samples: 53696 | elapsed time per iteration (ms): 223119.0 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.81% | FLOPs estimator: workload-aware | logical tokens/s: 75194.0 | valid tokens/s: 75194.0 | physical tokens/s: 75628.9 | loss tokens/s: 58019.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.324198E-01 | lm-global-supervised-tokens: 1.294528E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.179825E-02 | lm-token-buffer-mib: 2.335384E+02 | global_load_balancing_loss: 9.999282E-01 | loss scale: 1.0 | grad norm: 0.169 | params norm: 11162.785 | params rms: 0.060443 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:45:21.996465 | finish at 2026-10-17 19:05:55 + [2026-10-15 23:24:13] iteration 1679/ 2384 | consumed samples: 53728 | elapsed time per iteration (ms): 219876.3 | throughput per GPU (TFLOP/s/GPU): 25.8 | MFU 8.26% | FLOPs estimator: workload-aware | logical tokens/s: 76302.8 | valid tokens/s: 76302.8 | physical tokens/s: 76617.0 | loss tokens/s: 56818.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.519364E-01 | lm-global-supervised-tokens: 1.249295E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.839538E-02 | lm-token-buffer-mib: 2.335220E+02 | global_load_balancing_loss: 9.996280E-01 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11162.755 | params rms: 0.060443 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:03:32.793249 | finish at 2026-10-17 18:27:46 + [2026-10-15 23:28:25] iteration 1680/ 2384 | consumed samples: 53760 | elapsed time per iteration (ms): 252091.8 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.78% | FLOPs estimator: workload-aware | logical tokens/s: 66551.8 | valid tokens/s: 66551.8 | physical tokens/s: 68298.7 | loss tokens/s: 48320.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.562244E-01 | lm-global-supervised-tokens: 1.218109E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.322675E-02 | lm-token-buffer-mib: 2.409679E+02 | global_load_balancing_loss: 1.000170E+00 | loss scale: 1.0 | grad norm: 0.240 | params norm: 11162.723 | params rms: 0.060443 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:17:52.661926 | finish at 2026-10-18 00:46:18 +times across ranks (ms): + save-checkpoint: + rank 0: 23974.50 + rank 1: 23974.51 + rank 2: 23974.53 + rank 3: 23974.52 + rank 4: 23974.48 + rank 5: 23974.49 + rank 6: 23974.53 + rank 7: 23974.48 + rank 8: 23973.97 + rank 9: 23974.00 + rank 10: 23974.01 + rank 11: 23973.99 + rank 12: 23974.01 + rank 13: 23973.97 + rank 14: 23973.97 + rank 15: 23974.06 + rank 16: 23974.50 + rank 17: 23974.49 + rank 18: 23974.51 + rank 19: 23974.52 + rank 20: 23974.45 + rank 21: 23974.44 + rank 22: 23974.49 + rank 23: 23974.45 + rank 24: 23974.48 + rank 25: 23974.67 + rank 26: 23974.77 + rank 27: 23974.49 + rank 28: 23974.49 + rank 29: 23974.47 + rank 30: 23974.49 + rank 31: 23974.50 + rank 32: 23974.21 + rank 33: 23974.21 + rank 34: 23974.18 + rank 35: 23974.18 + rank 36: 23974.17 + rank 37: 23974.21 + rank 38: 23974.17 + rank 39: 23974.16 + rank 40: 23974.55 + rank 41: 23974.59 + rank 42: 23974.53 + rank 43: 23974.57 + rank 44: 23974.57 + rank 45: 23974.57 + rank 46: 23974.59 + rank 47: 23974.57 + rank 48: 23974.36 + rank 49: 23974.41 + rank 50: 23974.41 + rank 51: 23974.37 + rank 52: 23974.37 + rank 53: 23974.39 + rank 54: 23974.38 + rank 55: 23974.34 + rank 56: 23974.57 + rank 57: 23974.56 + rank 58: 23974.56 + rank 59: 23974.55 + rank 60: 23974.59 + rank 61: 23974.54 + rank 62: 23974.53 + rank 63: 23974.54 + [2026-10-15 23:32:31] iteration 1681/ 2384 | consumed samples: 53792 | elapsed time per iteration (ms): 221616.7 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.92% | FLOPs estimator: workload-aware | logical tokens/s: 75703.5 | valid tokens/s: 75703.5 | physical tokens/s: 76069.9 | loss tokens/s: 55373.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.156629E-01 | lm-global-supervised-tokens: 1.227164E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.475286E-02 | lm-token-buffer-mib: 2.335201E+02 | global_load_balancing_loss: 1.000979E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11162.689 | params rms: 0.060442 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:16:36.560502 | finish at 2026-10-17 18:49:07 + [2026-10-15 23:36:16] iteration 1682/ 2384 | consumed samples: 53824 | elapsed time per iteration (ms): 225075.4 | throughput per GPU (TFLOP/s/GPU): 25.8 | MFU 8.26% | FLOPs estimator: workload-aware | logical tokens/s: 74540.3 | valid tokens/s: 74540.3 | physical tokens/s: 74810.3 | loss tokens/s: 57119.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.504975E-01 | lm-global-supervised-tokens: 1.285619E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.568681E-02 | lm-token-buffer-mib: 2.334301E+02 | global_load_balancing_loss: 1.001300E+00 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11162.656 | params rms: 0.060442 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:53:22.907448 | finish at 2026-10-17 19:29:39 + [2026-10-15 23:39:57] iteration 1683/ 2384 | consumed samples: 53856 | elapsed time per iteration (ms): 221533.2 | throughput per GPU (TFLOP/s/GPU): 27.0 | MFU 8.65% | FLOPs estimator: workload-aware | logical tokens/s: 75732.3 | valid tokens/s: 75732.2 | physical tokens/s: 76518.4 | loss tokens/s: 54608.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.483983E-01 | lm-global-supervised-tokens: 1.209759E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.585705E-02 | lm-token-buffer-mib: 2.334001E+02 | global_load_balancing_loss: 1.001169E+00 | loss scale: 1.0 | grad norm: 0.293 | params norm: 11162.625 | params rms: 0.060442 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:08:14.773886 | finish at 2026-10-17 18:48:12 + [2026-10-15 23:43:39] iteration 1684/ 2384 | consumed samples: 53888 | elapsed time per iteration (ms): 222033.7 | throughput per GPU (TFLOP/s/GPU): 20.2 | MFU 6.49% | FLOPs estimator: workload-aware | logical tokens/s: 75561.5 | valid tokens/s: 75561.5 | physical tokens/s: 76562.9 | loss tokens/s: 60784.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.091931E-01 | lm-global-supervised-tokens: 1.349621E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.212465E-02 | lm-token-buffer-mib: 2.363809E+02 | global_load_balancing_loss: 1.000524E+00 | loss scale: 1.0 | grad norm: 0.170 | params norm: 11162.588 | params rms: 0.060442 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:10:23.599505 | finish at 2026-10-17 18:54:03 + [2026-10-15 23:47:23] iteration 1685/ 2384 | consumed samples: 53920 | elapsed time per iteration (ms): 223959.4 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.50% | FLOPs estimator: workload-aware | logical tokens/s: 74911.9 | valid tokens/s: 74911.9 | physical tokens/s: 75855.8 | loss tokens/s: 58425.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.376019E-01 | lm-global-supervised-tokens: 1.308492E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.893193E-02 | lm-token-buffer-mib: 2.359466E+02 | global_load_balancing_loss: 9.995196E-01 | loss scale: 1.0 | grad norm: 0.380 | params norm: 11162.551 | params rms: 0.060442 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:29:07.612224 | finish at 2026-10-17 19:16:31 + [2026-10-15 23:51:05] iteration 1686/ 2384 | consumed samples: 53952 | elapsed time per iteration (ms): 221713.0 | throughput per GPU (TFLOP/s/GPU): 25.5 | MFU 8.16% | FLOPs estimator: workload-aware | logical tokens/s: 75670.7 | valid tokens/s: 75670.7 | physical tokens/s: 76125.8 | loss tokens/s: 60410.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.651581E-01 | lm-global-supervised-tokens: 1.339384E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.213670E-02 | lm-token-buffer-mib: 2.338972E+02 | global_load_balancing_loss: 9.981304E-01 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11162.515 | params rms: 0.060441 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 18:59:15.699503 | finish at 2026-10-17 18:50:21 + [2026-10-15 23:54:46] iteration 1687/ 2384 | consumed samples: 53984 | elapsed time per iteration (ms): 220974.5 | throughput per GPU (TFLOP/s/GPU): 23.2 | MFU 7.43% | FLOPs estimator: workload-aware | logical tokens/s: 75923.5 | valid tokens/s: 75923.5 | physical tokens/s: 76517.2 | loss tokens/s: 59822.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.837573E-01 | lm-global-supervised-tokens: 1.321914E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.130222E-02 | lm-token-buffer-mib: 2.340303E+02 | global_load_balancing_loss: 9.997321E-01 | loss scale: 1.0 | grad norm: 0.164 | params norm: 11162.481 | params rms: 0.060441 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 18:46:59.212666 | finish at 2026-10-17 18:41:45 + [2026-10-15 23:58:28] iteration 1688/ 2384 | consumed samples: 54016 | elapsed time per iteration (ms): 222321.4 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.09% | FLOPs estimator: workload-aware | logical tokens/s: 75463.5 | valid tokens/s: 75463.5 | physical tokens/s: 76164.5 | loss tokens/s: 58255.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.574946E-01 | lm-global-supervised-tokens: 1.295153E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.908221E-02 | lm-token-buffer-mib: 2.347753E+02 | global_load_balancing_loss: 1.002226E+00 | loss scale: 1.0 | grad norm: 0.136 | params norm: 11162.447 | params rms: 0.060441 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 18:58:55.703642 | finish at 2026-10-17 18:57:24 + [2026-10-16 00:02:13] iteration 1689/ 2384 | consumed samples: 54048 | elapsed time per iteration (ms): 224067.1 | throughput per GPU (TFLOP/s/GPU): 19.9 | MFU 6.39% | FLOPs estimator: workload-aware | logical tokens/s: 74875.7 | valid tokens/s: 74875.7 | physical tokens/s: 75905.0 | loss tokens/s: 59102.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.296987E-01 | lm-global-supervised-tokens: 1.324295E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.271933E-02 | lm-token-buffer-mib: 2.354299E+02 | global_load_balancing_loss: 1.002574E+00 | loss scale: 1.0 | grad norm: 0.465 | params norm: 11162.411 | params rms: 0.060441 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:15:26.642321 | finish at 2026-10-17 19:17:39 + [2026-10-16 00:06:18] iteration 1690/ 2384 | consumed samples: 54080 | elapsed time per iteration (ms): 245858.4 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.39% | FLOPs estimator: workload-aware | logical tokens/s: 68239.1 | valid tokens/s: 68239.1 | physical tokens/s: 68668.9 | loss tokens/s: 54737.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.751009E-01 | lm-global-supervised-tokens: 1.345773E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.205898E-02 | lm-token-buffer-mib: 2.330885E+02 | global_load_balancing_loss: 1.001358E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11162.375 | params rms: 0.060441 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:23:45.698071 | finish at 2026-10-17 23:30:04 + [2026-10-16 00:10:00] iteration 1691/ 2384 | consumed samples: 54112 | elapsed time per iteration (ms): 222005.6 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.22% | FLOPs estimator: workload-aware | logical tokens/s: 75571.1 | valid tokens/s: 75571.1 | physical tokens/s: 76471.1 | loss tokens/s: 59404.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.717352E-01 | lm-global-supervised-tokens: 1.318807E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.913213E-02 | lm-token-buffer-mib: 2.346677E+02 | global_load_balancing_loss: 1.001776E+00 | loss scale: 1.0 | grad norm: 0.359 | params norm: 11162.338 | params rms: 0.060440 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 18:44:09.904080 | finish at 2026-10-17 18:54:10 + [2026-10-16 00:13:47] iteration 1692/ 2384 | consumed samples: 54144 | elapsed time per iteration (ms): 226369.5 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.70% | FLOPs estimator: workload-aware | logical tokens/s: 74114.1 | valid tokens/s: 74114.1 | physical tokens/s: 74653.7 | loss tokens/s: 50482.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.352727E-01 | lm-global-supervised-tokens: 1.142771E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.171239E-02 | lm-token-buffer-mib: 2.344780E+02 | global_load_balancing_loss: 9.996902E-01 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11162.300 | params rms: 0.060440 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:30:47.713579 | finish at 2026-10-17 19:44:34 + [2026-10-16 00:17:30] iteration 1693/ 2384 | consumed samples: 54176 | elapsed time per iteration (ms): 223693.4 | throughput per GPU (TFLOP/s/GPU): 19.0 | MFU 6.08% | FLOPs estimator: workload-aware | logical tokens/s: 75000.8 | valid tokens/s: 75000.8 | physical tokens/s: 76272.1 | loss tokens/s: 56664.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.856526E-01 | lm-global-supervised-tokens: 1.267555E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.252029E-02 | lm-token-buffer-mib: 2.367403E+02 | global_load_balancing_loss: 9.996975E-01 | loss scale: 1.0 | grad norm: 0.228 | params norm: 11162.261 | params rms: 0.060440 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 18:56:12.115447 | finish at 2026-10-17 19:13:43 + [2026-10-16 00:21:17] iteration 1694/ 2384 | consumed samples: 54208 | elapsed time per iteration (ms): 226523.1 | throughput per GPU (TFLOP/s/GPU): 24.9 | MFU 7.98% | FLOPs estimator: workload-aware | logical tokens/s: 74064.0 | valid tokens/s: 74064.0 | physical tokens/s: 74741.3 | loss tokens/s: 61091.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.452075E-01 | lm-global-supervised-tokens: 1.383862E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.459721E-02 | lm-token-buffer-mib: 2.353600E+02 | global_load_balancing_loss: 9.987934E-01 | loss scale: 1.0 | grad norm: 0.116 | params norm: 11162.221 | params rms: 0.060440 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:25:00.949788 | finish at 2026-10-17 19:46:18 + [2026-10-16 00:25:03] iteration 1695/ 2384 | consumed samples: 54240 | elapsed time per iteration (ms): 225761.5 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.41% | FLOPs estimator: workload-aware | logical tokens/s: 74313.8 | valid tokens/s: 74313.8 | physical tokens/s: 75159.2 | loss tokens/s: 58313.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.207661E-01 | lm-global-supervised-tokens: 1.316484E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.201838E-02 | lm-token-buffer-mib: 2.347117E+02 | global_load_balancing_loss: 9.984533E-01 | loss scale: 1.0 | grad norm: 0.211 | params norm: 11162.180 | params rms: 0.060440 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:12:29.667341 | finish at 2026-10-17 19:37:32 + [2026-10-16 00:28:47] iteration 1696/ 2384 | consumed samples: 54272 | elapsed time per iteration (ms): 224488.3 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.21% | FLOPs estimator: workload-aware | logical tokens/s: 74735.3 | valid tokens/s: 74735.3 | physical tokens/s: 75830.4 | loss tokens/s: 56518.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.325896E-01 | lm-global-supervised-tokens: 1.268765E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.115141E-02 | lm-token-buffer-mib: 2.343340E+02 | global_load_balancing_loss: 1.000334E+00 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11162.139 | params rms: 0.060439 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 18:54:07.921917 | finish at 2026-10-17 19:22:55 + [2026-10-16 00:32:29] iteration 1697/ 2384 | consumed samples: 54304 | elapsed time per iteration (ms): 221706.7 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.56% | FLOPs estimator: workload-aware | logical tokens/s: 75672.9 | valid tokens/s: 75672.9 | physical tokens/s: 76717.7 | loss tokens/s: 57392.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.016748E-01 | lm-global-supervised-tokens: 1.272420E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.930404E-02 | lm-token-buffer-mib: 2.341711E+02 | global_load_balancing_loss: 1.001051E+00 | loss scale: 1.0 | grad norm: 0.115 | params norm: 11162.094 | params rms: 0.060439 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 18:18:32.469709 | finish at 2026-10-17 18:51:01 + [2026-10-16 00:36:07] iteration 1698/ 2384 | consumed samples: 54336 | elapsed time per iteration (ms): 218469.7 | throughput per GPU (TFLOP/s/GPU): 19.7 | MFU 6.31% | FLOPs estimator: workload-aware | logical tokens/s: 76794.2 | valid tokens/s: 76794.2 | physical tokens/s: 77691.2 | loss tokens/s: 57513.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.547371E-01 | lm-global-supervised-tokens: 1.256493E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.327478E-02 | lm-token-buffer-mib: 2.335788E+02 | global_load_balancing_loss: 1.001351E+00 | loss scale: 1.0 | grad norm: 0.145 | params norm: 11162.051 | params rms: 0.060439 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:37:50.220646 | finish at 2026-10-17 18:13:58 + [2026-10-16 00:39:47] iteration 1699/ 2384 | consumed samples: 54368 | elapsed time per iteration (ms): 219811.3 | throughput per GPU (TFLOP/s/GPU): 23.4 | MFU 7.51% | FLOPs estimator: workload-aware | logical tokens/s: 76325.2 | valid tokens/s: 76325.2 | physical tokens/s: 76776.3 | loss tokens/s: 59111.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.477508E-01 | lm-global-supervised-tokens: 1.299340E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.028627E-02 | lm-token-buffer-mib: 2.337010E+02 | global_load_balancing_loss: 1.002331E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11162.014 | params rms: 0.060439 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:49:30.732688 | finish at 2026-10-17 18:29:18 + [2026-10-16 00:43:52] iteration 1700/ 2384 | consumed samples: 54400 | elapsed time per iteration (ms): 244968.8 | throughput per GPU (TFLOP/s/GPU): 19.5 | MFU 6.24% | FLOPs estimator: workload-aware | logical tokens/s: 68487.0 | valid tokens/s: 68487.0 | physical tokens/s: 69123.1 | loss tokens/s: 51790.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.239283E-01 | lm-global-supervised-tokens: 1.268713E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.836129E-02 | lm-token-buffer-mib: 2.352455E+02 | global_load_balancing_loss: 1.001150E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11161.976 | params rms: 0.060438 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:32:38.633154 | finish at 2026-10-17 23:16:31 + [2026-10-16 00:47:35] iteration 1701/ 2384 | consumed samples: 54432 | elapsed time per iteration (ms): 222788.4 | throughput per GPU (TFLOP/s/GPU): 26.3 | MFU 8.43% | FLOPs estimator: workload-aware | logical tokens/s: 75305.3 | valid tokens/s: 75305.3 | physical tokens/s: 75683.9 | loss tokens/s: 60583.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.008944E-01 | lm-global-supervised-tokens: 1.349721E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.201497E-02 | lm-token-buffer-mib: 2.328389E+02 | global_load_balancing_loss: 1.000096E+00 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11161.936 | params rms: 0.060438 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 18:16:04.502396 | finish at 2026-10-17 19:03:39 + [2026-10-16 00:51:19] iteration 1702/ 2384 | consumed samples: 54464 | elapsed time per iteration (ms): 224401.4 | throughput per GPU (TFLOP/s/GPU): 24.2 | MFU 7.76% | FLOPs estimator: workload-aware | logical tokens/s: 74764.2 | valid tokens/s: 74764.2 | physical tokens/s: 75804.7 | loss tokens/s: 61289.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.037896E-01 | lm-global-supervised-tokens: 1.375333E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.355272E-02 | lm-token-buffer-mib: 2.373205E+02 | global_load_balancing_loss: 9.998179E-01 | loss scale: 1.0 | grad norm: 0.112 | params norm: 11161.894 | params rms: 0.060438 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 18:30:41.746406 | finish at 2026-10-17 19:22:01 + [2026-10-16 00:55:06] iteration 1703/ 2384 | consumed samples: 54496 | elapsed time per iteration (ms): 226380.6 | throughput per GPU (TFLOP/s/GPU): 24.1 | MFU 7.73% | FLOPs estimator: workload-aware | logical tokens/s: 74110.5 | valid tokens/s: 74110.5 | physical tokens/s: 75202.8 | loss tokens/s: 55939.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.292951E-01 | lm-global-supervised-tokens: 1.266350E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.553890E-02 | lm-token-buffer-mib: 2.365249E+02 | global_load_balancing_loss: 9.999705E-01 | loss scale: 1.0 | grad norm: 0.138 | params norm: 11161.848 | params rms: 0.060438 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 18:49:25.156760 | finish at 2026-10-17 19:44:31 + [2026-10-16 00:58:50] iteration 1704/ 2384 | consumed samples: 54528 | elapsed time per iteration (ms): 224413.3 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.19% | FLOPs estimator: workload-aware | logical tokens/s: 74760.3 | valid tokens/s: 74760.3 | physical tokens/s: 76020.9 | loss tokens/s: 56273.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.392819E-01 | lm-global-supervised-tokens: 1.262843E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.487647E-02 | lm-token-buffer-mib: 2.374680E+02 | global_load_balancing_loss: 9.998570E-01 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11161.804 | params rms: 0.060438 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 18:23:21.076937 | finish at 2026-10-17 19:22:11 + [2026-10-16 01:02:31] iteration 1705/ 2384 | consumed samples: 54560 | elapsed time per iteration (ms): 220916.1 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.81% | FLOPs estimator: workload-aware | logical tokens/s: 75943.7 | valid tokens/s: 75943.7 | physical tokens/s: 76793.5 | loss tokens/s: 58949.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.219961E-01 | lm-global-supervised-tokens: 1.302299E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.161149E-02 | lm-token-buffer-mib: 2.338957E+02 | global_load_balancing_loss: 9.988170E-01 | loss scale: 1.0 | grad norm: 0.104 | params norm: 11161.766 | params rms: 0.060437 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:40:02.021071 | finish at 2026-10-17 18:42:33 + [2026-10-16 01:06:17] iteration 1706/ 2384 | consumed samples: 54592 | elapsed time per iteration (ms): 225669.6 | throughput per GPU (TFLOP/s/GPU): 23.4 | MFU 7.49% | FLOPs estimator: workload-aware | logical tokens/s: 74208.3 | valid tokens/s: 74208.3 | physical tokens/s: 74931.2 | loss tokens/s: 52728.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.725018E-01 | lm-global-supervised-tokens: 1.189923E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.894470E-02 | lm-token-buffer-mib: 2.342493E+02 | global_load_balancing_loss: 1.000614E+00 | loss scale: 1.0 | grad norm: 0.269 | params norm: 11161.731 | params rms: 0.060437 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 18:30:04.021621 | finish at 2026-10-17 19:36:21 + [2026-10-16 01:10:00] iteration 1707/ 2384 | consumed samples: 54624 | elapsed time per iteration (ms): 222888.6 | throughput per GPU (TFLOP/s/GPU): 19.9 | MFU 6.38% | FLOPs estimator: workload-aware | logical tokens/s: 75271.7 | valid tokens/s: 75271.7 | physical tokens/s: 76480.3 | loss tokens/s: 61261.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.032213E-01 | lm-global-supervised-tokens: 1.365449E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.128936E-02 | lm-token-buffer-mib: 2.353397E+02 | global_load_balancing_loss: 9.994271E-01 | loss scale: 1.0 | grad norm: 0.210 | params norm: 11161.691 | params rms: 0.060437 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:54:55.554997 | finish at 2026-10-17 19:04:55 + [2026-10-16 01:13:40] iteration 1708/ 2384 | consumed samples: 54656 | elapsed time per iteration (ms): 220556.8 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.45% | FLOPs estimator: workload-aware | logical tokens/s: 76067.5 | valid tokens/s: 76067.5 | physical tokens/s: 76781.1 | loss tokens/s: 57312.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.415860E-01 | lm-global-supervised-tokens: 1.264057E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.250228E-02 | lm-token-buffer-mib: 2.329908E+02 | global_load_balancing_loss: 1.001000E+00 | loss scale: 1.0 | grad norm: 0.157 | params norm: 11161.651 | params rms: 0.060437 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:24:56.398981 | finish at 2026-10-17 18:38:37 + [2026-10-16 01:17:21] iteration 1709/ 2384 | consumed samples: 54688 | elapsed time per iteration (ms): 220606.2 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.19% | FLOPs estimator: workload-aware | logical tokens/s: 76050.4 | valid tokens/s: 76050.3 | physical tokens/s: 76742.2 | loss tokens/s: 58021.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.247398E-01 | lm-global-supervised-tokens: 1.279998E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.106221E-02 | lm-token-buffer-mib: 2.330046E+02 | global_load_balancing_loss: 1.001390E+00 | loss scale: 1.0 | grad norm: 0.100 | params norm: 11161.612 | params rms: 0.060436 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:21:49.158915 | finish at 2026-10-17 18:39:10 + [2026-10-16 01:21:27] iteration 1710/ 2384 | consumed samples: 54720 | elapsed time per iteration (ms): 246660.7 | throughput per GPU (TFLOP/s/GPU): 19.4 | MFU 6.21% | FLOPs estimator: workload-aware | logical tokens/s: 68017.3 | valid tokens/s: 68017.3 | physical tokens/s: 68581.7 | loss tokens/s: 49233.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.723333E-01 | lm-global-supervised-tokens: 1.214388E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.109888E-02 | lm-token-buffer-mib: 2.341688E+02 | global_load_balancing_loss: 1.001505E+00 | loss scale: 1.0 | grad norm: 0.168 | params norm: 11161.575 | params rms: 0.060436 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:10:49.312338 | finish at 2026-10-17 23:32:17 + [2026-10-16 01:25:08] iteration 1711/ 2384 | consumed samples: 54752 | elapsed time per iteration (ms): 220153.3 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.87% | FLOPs estimator: workload-aware | logical tokens/s: 76206.9 | valid tokens/s: 76206.9 | physical tokens/s: 76974.7 | loss tokens/s: 53691.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.196325E-01 | lm-global-supervised-tokens: 1.182027E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.464868E-02 | lm-token-buffer-mib: 2.333757E+02 | global_load_balancing_loss: 1.002456E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11161.540 | params rms: 0.060436 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:09:23.147666 | finish at 2026-10-17 18:34:31 + [2026-10-16 01:28:49] iteration 1712/ 2384 | consumed samples: 54784 | elapsed time per iteration (ms): 221175.7 | throughput per GPU (TFLOP/s/GPU): 25.0 | MFU 8.00% | FLOPs estimator: workload-aware | logical tokens/s: 75854.5 | valid tokens/s: 75854.5 | physical tokens/s: 76293.6 | loss tokens/s: 55301.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.707271E-01 | lm-global-supervised-tokens: 1.223138E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.189018E-02 | lm-token-buffer-mib: 2.328605E+02 | global_load_balancing_loss: 1.000705E+00 | loss scale: 1.0 | grad norm: 0.117 | params norm: 11161.505 | params rms: 0.060436 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:17:10.095680 | finish at 2026-10-17 18:45:59 + [2026-10-16 01:32:28] iteration 1713/ 2384 | consumed samples: 54816 | elapsed time per iteration (ms): 219609.5 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.56% | FLOPs estimator: workload-aware | logical tokens/s: 76395.7 | valid tokens/s: 76395.6 | physical tokens/s: 77298.8 | loss tokens/s: 60912.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.505609E-01 | lm-global-supervised-tokens: 1.337700E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.266880E-02 | lm-token-buffer-mib: 2.341211E+02 | global_load_balancing_loss: 1.000026E+00 | loss scale: 1.0 | grad norm: 0.110 | params norm: 11161.468 | params rms: 0.060436 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:55:57.962935 | finish at 2026-10-17 18:28:26 + [2026-10-16 01:36:09] iteration 1714/ 2384 | consumed samples: 54848 | elapsed time per iteration (ms): 220886.4 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.57% | FLOPs estimator: workload-aware | logical tokens/s: 75953.9 | valid tokens/s: 75953.9 | physical tokens/s: 76352.4 | loss tokens/s: 56443.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.674317E-01 | lm-global-supervised-tokens: 1.246770E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.628762E-02 | lm-token-buffer-mib: 2.336378E+02 | global_load_balancing_loss: 9.979720E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11161.433 | params rms: 0.060436 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:06:33.914187 | finish at 2026-10-17 18:42:43 + [2026-10-16 01:39:52] iteration 1715/ 2384 | consumed samples: 54880 | elapsed time per iteration (ms): 222797.6 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.57% | FLOPs estimator: workload-aware | logical tokens/s: 75302.3 | valid tokens/s: 75302.3 | physical tokens/s: 75939.3 | loss tokens/s: 58405.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.204337E-01 | lm-global-supervised-tokens: 1.301252E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.379463E-02 | lm-token-buffer-mib: 2.326416E+02 | global_load_balancing_loss: 9.982745E-01 | loss scale: 1.0 | grad norm: 0.255 | params norm: 11161.398 | params rms: 0.060435 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:24:11.624088 | finish at 2026-10-17 19:04:04 + [2026-10-16 01:43:37] iteration 1716/ 2384 | consumed samples: 54912 | elapsed time per iteration (ms): 224629.1 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.04% | FLOPs estimator: workload-aware | logical tokens/s: 74688.5 | valid tokens/s: 74688.5 | physical tokens/s: 75880.9 | loss tokens/s: 59950.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.063959E-01 | lm-global-supervised-tokens: 1.346669E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.263969E-02 | lm-token-buffer-mib: 2.369012E+02 | global_load_balancing_loss: 9.996238E-01 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11161.365 | params rms: 0.060435 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:40:52.218630 | finish at 2026-10-17 19:24:29 + [2026-10-16 01:47:21] iteration 1717/ 2384 | consumed samples: 54944 | elapsed time per iteration (ms): 224691.0 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.95% | FLOPs estimator: workload-aware | logical tokens/s: 74667.9 | valid tokens/s: 74667.9 | physical tokens/s: 75249.0 | loss tokens/s: 55977.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.859986E-01 | lm-global-supervised-tokens: 1.257766E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.197402E-02 | lm-token-buffer-mib: 2.334531E+02 | global_load_balancing_loss: 1.000941E+00 | loss scale: 1.0 | grad norm: 0.224 | params norm: 11161.332 | params rms: 0.060435 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:37:48.873295 | finish at 2026-10-17 19:25:10 + [2026-10-16 01:51:02] iteration 1718/ 2384 | consumed samples: 54976 | elapsed time per iteration (ms): 221069.9 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.42% | FLOPs estimator: workload-aware | logical tokens/s: 75891.0 | valid tokens/s: 75891.0 | physical tokens/s: 76750.0 | loss tokens/s: 60636.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.543672E-01 | lm-global-supervised-tokens: 1.340482E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.169366E-02 | lm-token-buffer-mib: 2.353416E+02 | global_load_balancing_loss: 1.000994E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11161.297 | params rms: 0.060435 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:53:52.534369 | finish at 2026-10-17 18:44:55 + [2026-10-16 01:54:45] iteration 1719/ 2384 | consumed samples: 55008 | elapsed time per iteration (ms): 222418.8 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.90% | FLOPs estimator: workload-aware | logical tokens/s: 75430.6 | valid tokens/s: 75430.6 | physical tokens/s: 76356.7 | loss tokens/s: 54782.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.606375E-01 | lm-global-supervised-tokens: 1.218474E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.689241E-02 | lm-token-buffer-mib: 2.326953E+02 | global_load_balancing_loss: 1.002543E+00 | loss scale: 1.0 | grad norm: 0.161 | params norm: 11161.264 | params rms: 0.060435 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:05:08.506516 | finish at 2026-10-17 18:59:53 + [2026-10-16 01:58:54] iteration 1720/ 2384 | consumed samples: 55040 | elapsed time per iteration (ms): 248989.7 | throughput per GPU (TFLOP/s/GPU): 18.6 | MFU 5.95% | FLOPs estimator: workload-aware | logical tokens/s: 67381.2 | valid tokens/s: 67381.2 | physical tokens/s: 68302.1 | loss tokens/s: 48149.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.126847E-01 | lm-global-supervised-tokens: 1.198879E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.688941E-02 | lm-token-buffer-mib: 2.352474E+02 | global_load_balancing_loss: 1.000570E+00 | loss scale: 1.0 | grad norm: 0.251 | params norm: 11161.231 | params rms: 0.060434 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:55:29.147713 | finish at 2026-10-17 23:54:23 +times across ranks (ms): + save-checkpoint: + rank 0: 28447.34 + rank 1: 28447.42 + rank 2: 28447.35 + rank 3: 28447.35 + rank 4: 28447.33 + rank 5: 28447.36 + rank 6: 28447.35 + rank 7: 28447.27 + rank 8: 28449.71 + rank 9: 28449.71 + rank 10: 28449.75 + rank 11: 28449.69 + rank 12: 28449.73 + rank 13: 28449.68 + rank 14: 28449.77 + rank 15: 28449.69 + rank 16: 28447.40 + rank 17: 28447.39 + rank 18: 28447.50 + rank 19: 28447.38 + rank 20: 28447.35 + rank 21: 28447.37 + rank 22: 28447.41 + rank 23: 28447.20 + rank 24: 28448.30 + rank 25: 28448.31 + rank 26: 28448.34 + rank 27: 28448.97 + rank 28: 28448.26 + rank 29: 28448.33 + rank 30: 28448.28 + rank 31: 28448.28 + rank 32: 28449.37 + rank 33: 28449.44 + rank 34: 28449.42 + rank 35: 28449.46 + rank 36: 28449.40 + rank 37: 28449.45 + rank 38: 28449.39 + rank 39: 28449.40 + rank 40: 28447.49 + rank 41: 28447.47 + rank 42: 28447.47 + rank 43: 28447.48 + rank 44: 28447.46 + rank 45: 28447.46 + rank 46: 28447.50 + rank 47: 28447.46 + rank 48: 28446.19 + rank 49: 28446.16 + rank 50: 28446.20 + rank 51: 28446.06 + rank 52: 28446.07 + rank 53: 28446.18 + rank 54: 28446.14 + rank 55: 28446.11 + rank 56: 28447.90 + rank 57: 28448.01 + rank 58: 28447.98 + rank 59: 28447.94 + rank 60: 28447.94 + rank 61: 28447.88 + rank 62: 28447.99 + rank 63: 28447.89 + [2026-10-16 02:03:23] iteration 1721/ 2384 | consumed samples: 55072 | elapsed time per iteration (ms): 238697.4 | throughput per GPU (TFLOP/s/GPU): 19.3 | MFU 6.18% | FLOPs estimator: workload-aware | logical tokens/s: 70286.5 | valid tokens/s: 70286.4 | physical tokens/s: 70853.9 | loss tokens/s: 54868.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.530630E-01 | lm-global-supervised-tokens: 1.309699E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 6.424604E+00 | lm-token-buffer-mib: 2.339404E+02 | global_load_balancing_loss: 1.000747E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11161.198 | params rms: 0.060434 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:57:36.383533 | finish at 2026-10-17 22:00:59 + [2026-10-16 02:07:10] iteration 1722/ 2384 | consumed samples: 55104 | elapsed time per iteration (ms): 227178.5 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.81% | FLOPs estimator: workload-aware | logical tokens/s: 73850.3 | valid tokens/s: 73850.3 | physical tokens/s: 74676.6 | loss tokens/s: 56314.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.983872E-01 | lm-global-supervised-tokens: 1.279333E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.863575E-02 | lm-token-buffer-mib: 2.356373E+02 | global_load_balancing_loss: 1.000954E+00 | loss scale: 1.0 | grad norm: 0.363 | params norm: 11161.167 | params rms: 0.060434 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:46:32.169326 | finish at 2026-10-17 19:53:42 + [2026-10-16 02:10:49] iteration 1723/ 2384 | consumed samples: 55136 | elapsed time per iteration (ms): 218608.2 | throughput per GPU (TFLOP/s/GPU): 19.3 | MFU 6.20% | FLOPs estimator: workload-aware | logical tokens/s: 76745.5 | valid tokens/s: 76745.5 | physical tokens/s: 77794.9 | loss tokens/s: 59003.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.864420E-01 | lm-global-supervised-tokens: 1.289866E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.024573E-02 | lm-token-buffer-mib: 2.345781E+02 | global_load_balancing_loss: 9.997028E-01 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11161.140 | params rms: 0.060434 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:08:19.995979 | finish at 2026-10-17 18:19:09 + [2026-10-16 02:14:30] iteration 1724/ 2384 | consumed samples: 55168 | elapsed time per iteration (ms): 221934.8 | throughput per GPU (TFLOP/s/GPU): 23.2 | MFU 7.44% | FLOPs estimator: workload-aware | logical tokens/s: 75595.1 | valid tokens/s: 75595.1 | physical tokens/s: 76240.9 | loss tokens/s: 53366.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.972216E-01 | lm-global-supervised-tokens: 1.184386E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.639260E-02 | lm-token-buffer-mib: 2.354642E+02 | global_load_balancing_loss: 9.989581E-01 | loss scale: 1.0 | grad norm: 0.212 | params norm: 11161.110 | params rms: 0.060434 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:41:16.938043 | finish at 2026-10-17 18:55:47 + [2026-10-16 02:18:13] iteration 1725/ 2384 | consumed samples: 55200 | elapsed time per iteration (ms): 222341.9 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.12% | FLOPs estimator: workload-aware | logical tokens/s: 75456.6 | valid tokens/s: 75456.6 | physical tokens/s: 76067.1 | loss tokens/s: 56496.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.578880E-01 | lm-global-supervised-tokens: 1.256155E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.484191E-02 | lm-token-buffer-mib: 2.334244E+02 | global_load_balancing_loss: 9.996052E-01 | loss scale: 1.0 | grad norm: 0.231 | params norm: 11161.077 | params rms: 0.060434 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:42:03.315629 | finish at 2026-10-17 19:00:16 + [2026-10-16 02:21:57] iteration 1726/ 2384 | consumed samples: 55232 | elapsed time per iteration (ms): 223963.6 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.54% | FLOPs estimator: workload-aware | logical tokens/s: 74910.4 | valid tokens/s: 74910.4 | physical tokens/s: 76057.8 | loss tokens/s: 58337.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.470901E-01 | lm-global-supervised-tokens: 1.306558E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.192474E-02 | lm-token-buffer-mib: 2.347613E+02 | global_load_balancing_loss: 9.996039E-01 | loss scale: 1.0 | grad norm: 0.145 | params norm: 11161.047 | params rms: 0.060433 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:56:08.036197 | finish at 2026-10-17 19:18:05 + [2026-10-16 02:25:41] iteration 1727/ 2384 | consumed samples: 55264 | elapsed time per iteration (ms): 224172.2 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.95% | FLOPs estimator: workload-aware | logical tokens/s: 74840.6 | valid tokens/s: 74840.6 | physical tokens/s: 75582.8 | loss tokens/s: 54940.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.155382E-01 | lm-global-supervised-tokens: 1.231612E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.681945E-02 | lm-token-buffer-mib: 2.352529E+02 | global_load_balancing_loss: 9.992141E-01 | loss scale: 1.0 | grad norm: 0.220 | params norm: 11161.018 | params rms: 0.060433 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:54:41.140233 | finish at 2026-10-17 19:20:22 + [2026-10-16 02:29:22] iteration 1728/ 2384 | consumed samples: 55296 | elapsed time per iteration (ms): 221536.4 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.64% | FLOPs estimator: workload-aware | logical tokens/s: 75731.1 | valid tokens/s: 75731.1 | physical tokens/s: 77396.4 | loss tokens/s: 57579.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.259803E-01 | lm-global-supervised-tokens: 1.275605E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.066180E-02 | lm-token-buffer-mib: 2.363457E+02 | global_load_balancing_loss: 1.000201E+00 | loss scale: 1.0 | grad norm: 0.322 | params norm: 11160.988 | params rms: 0.060433 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:22:07.865627 | finish at 2026-10-17 18:51:30 + [2026-10-16 02:33:06] iteration 1729/ 2384 | consumed samples: 55328 | elapsed time per iteration (ms): 223130.3 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.14% | FLOPs estimator: workload-aware | logical tokens/s: 75190.1 | valid tokens/s: 75190.1 | physical tokens/s: 75748.4 | loss tokens/s: 55273.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.528163E-01 | lm-global-supervised-tokens: 1.233313E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.578725E-02 | lm-token-buffer-mib: 2.341932E+02 | global_load_balancing_loss: 1.001747E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11160.955 | params rms: 0.060433 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:35:50.344187 | finish at 2026-10-17 19:08:56 + [2026-10-16 02:37:09] iteration 1730/ 2384 | consumed samples: 55360 | elapsed time per iteration (ms): 243578.3 | throughput per GPU (TFLOP/s/GPU): 17.9 | MFU 5.75% | FLOPs estimator: workload-aware | logical tokens/s: 68878.1 | valid tokens/s: 68878.1 | physical tokens/s: 69478.5 | loss tokens/s: 53812.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.719870E-01 | lm-global-supervised-tokens: 1.310759E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.592072E-02 | lm-token-buffer-mib: 2.351917E+02 | global_load_balancing_loss: 1.000140E+00 | loss scale: 1.0 | grad norm: 0.267 | params norm: 11160.921 | params rms: 0.060433 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:15:00.224884 | finish at 2026-10-17 22:52:09 + [2026-10-16 02:40:52] iteration 1731/ 2384 | consumed samples: 55392 | elapsed time per iteration (ms): 222646.7 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.78% | FLOPs estimator: workload-aware | logical tokens/s: 75353.4 | valid tokens/s: 75353.4 | physical tokens/s: 76182.8 | loss tokens/s: 58799.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.709157E-01 | lm-global-supervised-tokens: 1.309148E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.908706E-02 | lm-token-buffer-mib: 2.345879E+02 | global_load_balancing_loss: 1.002055E+00 | loss scale: 1.0 | grad norm: 0.196 | params norm: 11160.887 | params rms: 0.060433 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:23:08.267637 | finish at 2026-10-17 19:04:00 + [2026-10-16 02:44:36] iteration 1732/ 2384 | consumed samples: 55424 | elapsed time per iteration (ms): 223917.6 | throughput per GPU (TFLOP/s/GPU): 23.4 | MFU 7.50% | FLOPs estimator: workload-aware | logical tokens/s: 74925.8 | valid tokens/s: 74925.8 | physical tokens/s: 75728.4 | loss tokens/s: 60791.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.505582E-01 | lm-global-supervised-tokens: 1.361219E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.088487E-02 | lm-token-buffer-mib: 2.344486E+02 | global_load_balancing_loss: 1.002150E+00 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11160.857 | params rms: 0.060432 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:33:14.249807 | finish at 2026-10-17 19:17:50 + [2026-10-16 02:48:19] iteration 1733/ 2384 | consumed samples: 55456 | elapsed time per iteration (ms): 223255.3 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.10% | FLOPs estimator: workload-aware | logical tokens/s: 75148.1 | valid tokens/s: 75148.1 | physical tokens/s: 75476.4 | loss tokens/s: 56336.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.522666E-01 | lm-global-supervised-tokens: 1.257750E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.219016E-02 | lm-token-buffer-mib: 2.330097E+02 | global_load_balancing_loss: 1.001081E+00 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11160.826 | params rms: 0.060432 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:22:19.171460 | finish at 2026-10-17 19:10:38 + [2026-10-16 02:52:04] iteration 1734/ 2384 | consumed samples: 55488 | elapsed time per iteration (ms): 224755.2 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.76% | FLOPs estimator: workload-aware | logical tokens/s: 74646.5 | valid tokens/s: 74646.5 | physical tokens/s: 75445.1 | loss tokens/s: 56044.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.373978E-01 | lm-global-supervised-tokens: 1.259631E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.697266E-02 | lm-token-buffer-mib: 2.329830E+02 | global_load_balancing_loss: 9.990501E-01 | loss scale: 1.0 | grad norm: 0.232 | params norm: 11160.792 | params rms: 0.060432 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:34:50.873587 | finish at 2026-10-17 19:26:55 + [2026-10-16 02:55:45] iteration 1735/ 2384 | consumed samples: 55520 | elapsed time per iteration (ms): 221587.5 | throughput per GPU (TFLOP/s/GPU): 23.8 | MFU 7.64% | FLOPs estimator: workload-aware | logical tokens/s: 75713.6 | valid tokens/s: 75713.6 | physical tokens/s: 76078.8 | loss tokens/s: 55498.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.552482E-01 | lm-global-supervised-tokens: 1.229770E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.355381E-02 | lm-token-buffer-mib: 2.327801E+02 | global_load_balancing_loss: 9.990773E-01 | loss scale: 1.0 | grad norm: 0.138 | params norm: 11160.756 | params rms: 0.060432 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:56:50.281527 | finish at 2026-10-17 18:52:36 + [2026-10-16 02:59:28] iteration 1736/ 2384 | consumed samples: 55552 | elapsed time per iteration (ms): 222297.0 | throughput per GPU (TFLOP/s/GPU): 26.1 | MFU 8.37% | FLOPs estimator: workload-aware | logical tokens/s: 75471.8 | valid tokens/s: 75471.8 | physical tokens/s: 76507.9 | loss tokens/s: 53591.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.454513E-01 | lm-global-supervised-tokens: 1.191328E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.199892E-02 | lm-token-buffer-mib: 2.325740E+02 | global_load_balancing_loss: 1.000181E+00 | loss scale: 1.0 | grad norm: 0.166 | params norm: 11160.721 | params rms: 0.060432 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:00:48.486700 | finish at 2026-10-17 19:00:16 + [2026-10-16 03:03:10] iteration 1737/ 2384 | consumed samples: 55584 | elapsed time per iteration (ms): 222127.6 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.82% | FLOPs estimator: workload-aware | logical tokens/s: 75529.6 | valid tokens/s: 75529.6 | physical tokens/s: 76733.1 | loss tokens/s: 56213.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.950751E-01 | lm-global-supervised-tokens: 1.248660E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.218976E-02 | lm-token-buffer-mib: 2.324406E+02 | global_load_balancing_loss: 1.000355E+00 | loss scale: 1.0 | grad norm: 0.196 | params norm: 11160.684 | params rms: 0.060431 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:55:16.567197 | finish at 2026-10-17 18:58:26 + [2026-10-16 03:06:50] iteration 1738/ 2384 | consumed samples: 55616 | elapsed time per iteration (ms): 220368.3 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.34% | FLOPs estimator: workload-aware | logical tokens/s: 76132.5 | valid tokens/s: 76132.5 | physical tokens/s: 76966.7 | loss tokens/s: 58500.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.714478E-01 | lm-global-supervised-tokens: 1.289165E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.807414E-02 | lm-token-buffer-mib: 2.342694E+02 | global_load_balancing_loss: 9.993739E-01 | loss scale: 1.0 | grad norm: 0.181 | params norm: 11160.647 | params rms: 0.060431 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:32:37.931170 | finish at 2026-10-17 18:39:28 + [2026-10-16 03:10:31] iteration 1739/ 2384 | consumed samples: 55648 | elapsed time per iteration (ms): 220699.6 | throughput per GPU (TFLOP/s/GPU): 24.5 | MFU 7.86% | FLOPs estimator: workload-aware | logical tokens/s: 76018.0 | valid tokens/s: 76018.0 | physical tokens/s: 76471.0 | loss tokens/s: 57461.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.805227E-01 | lm-global-supervised-tokens: 1.268177E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.147379E-02 | lm-token-buffer-mib: 2.325810E+02 | global_load_balancing_loss: 9.991606E-01 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11160.610 | params rms: 0.060431 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:32:31.217229 | finish at 2026-10-17 18:43:02 + [2026-10-16 03:14:34] iteration 1740/ 2384 | consumed samples: 55680 | elapsed time per iteration (ms): 242884.9 | throughput per GPU (TFLOP/s/GPU): 16.5 | MFU 5.28% | FLOPs estimator: workload-aware | logical tokens/s: 69074.7 | valid tokens/s: 69074.7 | physical tokens/s: 69701.4 | loss tokens/s: 52176.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.984638E-01 | lm-global-supervised-tokens: 1.267292E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.623692E-02 | lm-token-buffer-mib: 2.336938E+02 | global_load_balancing_loss: 9.997537E-01 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11160.572 | params rms: 0.060431 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:26:57.855700 | finish at 2026-10-17 22:41:32 + [2026-10-16 03:18:16] iteration 1741/ 2384 | consumed samples: 55712 | elapsed time per iteration (ms): 222657.3 | throughput per GPU (TFLOP/s/GPU): 23.5 | MFU 7.54% | FLOPs estimator: workload-aware | logical tokens/s: 75349.7 | valid tokens/s: 75349.7 | physical tokens/s: 75938.3 | loss tokens/s: 58646.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.586431E-01 | lm-global-supervised-tokens: 1.305808E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.332082E-02 | lm-token-buffer-mib: 2.339619E+02 | global_load_balancing_loss: 1.001350E+00 | loss scale: 1.0 | grad norm: 0.177 | params norm: 11160.535 | params rms: 0.060431 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:46:08.658001 | finish at 2026-10-17 19:04:25 + [2026-10-16 03:22:00] iteration 1742/ 2384 | consumed samples: 55744 | elapsed time per iteration (ms): 223411.8 | throughput per GPU (TFLOP/s/GPU): 27.3 | MFU 8.77% | FLOPs estimator: workload-aware | logical tokens/s: 75095.4 | valid tokens/s: 75095.4 | physical tokens/s: 75788.2 | loss tokens/s: 54601.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.642748E-01 | lm-global-supervised-tokens: 1.219863E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.055587E-02 | lm-token-buffer-mib: 2.337624E+02 | global_load_balancing_loss: 1.003747E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11160.497 | params rms: 0.060430 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:50:30.379214 | finish at 2026-10-17 19:12:30 + [2026-10-16 03:25:46] iteration 1743/ 2384 | consumed samples: 55776 | elapsed time per iteration (ms): 225841.6 | throughput per GPU (TFLOP/s/GPU): 19.6 | MFU 6.27% | FLOPs estimator: workload-aware | logical tokens/s: 74287.4 | valid tokens/s: 74287.4 | physical tokens/s: 75433.8 | loss tokens/s: 58936.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.556256E-01 | lm-global-supervised-tokens: 1.331020E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.320241E-02 | lm-token-buffer-mib: 2.378097E+02 | global_load_balancing_loss: 1.001445E+00 | loss scale: 1.0 | grad norm: 0.103 | params norm: 11160.461 | params rms: 0.060430 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:12:44.483596 | finish at 2026-10-17 19:38:30 + [2026-10-16 03:29:27] iteration 1744/ 2384 | consumed samples: 55808 | elapsed time per iteration (ms): 221516.2 | throughput per GPU (TFLOP/s/GPU): 24.2 | MFU 7.75% | FLOPs estimator: workload-aware | logical tokens/s: 75738.0 | valid tokens/s: 75738.0 | physical tokens/s: 76926.1 | loss tokens/s: 55674.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.394021E-01 | lm-global-supervised-tokens: 1.233288E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.725981E-02 | lm-token-buffer-mib: 2.329971E+02 | global_load_balancing_loss: 1.001336E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11160.428 | params rms: 0.060430 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:22:50.354767 | finish at 2026-10-17 18:52:18 + [2026-10-16 03:33:12] iteration 1745/ 2384 | consumed samples: 55840 | elapsed time per iteration (ms): 225088.7 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.18% | FLOPs estimator: workload-aware | logical tokens/s: 74536.0 | valid tokens/s: 74536.0 | physical tokens/s: 75419.7 | loss tokens/s: 57842.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.458302E-01 | lm-global-supervised-tokens: 1.301969E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.316720E-02 | lm-token-buffer-mib: 2.348785E+02 | global_load_balancing_loss: 1.000788E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11160.392 | params rms: 0.060430 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:57:11.678422 | finish at 2026-10-17 19:30:24 + [2026-10-16 03:36:57] iteration 1746/ 2384 | consumed samples: 55872 | elapsed time per iteration (ms): 224748.4 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.07% | FLOPs estimator: workload-aware | logical tokens/s: 74648.8 | valid tokens/s: 74648.8 | physical tokens/s: 75149.5 | loss tokens/s: 56854.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.739823E-01 | lm-global-supervised-tokens: 1.277803E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.699254E-02 | lm-token-buffer-mib: 2.336664E+02 | global_load_balancing_loss: 9.996288E-01 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11160.358 | params rms: 0.060430 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:49:49.465645 | finish at 2026-10-17 19:26:46 + [2026-10-16 03:40:42] iteration 1747/ 2384 | consumed samples: 55904 | elapsed time per iteration (ms): 225508.0 | throughput per GPU (TFLOP/s/GPU): 25.8 | MFU 8.26% | FLOPs estimator: workload-aware | logical tokens/s: 74397.3 | valid tokens/s: 74397.3 | physical tokens/s: 74956.0 | loss tokens/s: 58067.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.910448E-01 | lm-global-supervised-tokens: 1.309470E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.914577E-02 | lm-token-buffer-mib: 2.342965E+02 | global_load_balancing_loss: 9.999866E-01 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11160.325 | params rms: 0.060430 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:54:08.607021 | finish at 2026-10-17 19:34:51 + [2026-10-16 03:44:27] iteration 1748/ 2384 | consumed samples: 55936 | elapsed time per iteration (ms): 224331.6 | throughput per GPU (TFLOP/s/GPU): 19.4 | MFU 6.22% | FLOPs estimator: workload-aware | logical tokens/s: 74787.5 | valid tokens/s: 74787.5 | physical tokens/s: 75813.2 | loss tokens/s: 60355.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.577691E-01 | lm-global-supervised-tokens: 1.353955E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.538402E-02 | lm-token-buffer-mib: 2.355454E+02 | global_load_balancing_loss: 1.000068E+00 | loss scale: 1.0 | grad norm: 0.109 | params norm: 11160.293 | params rms: 0.060429 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:37:54.871943 | finish at 2026-10-17 19:22:22 + [2026-10-16 03:48:07] iteration 1749/ 2384 | consumed samples: 55968 | elapsed time per iteration (ms): 220240.5 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.43% | FLOPs estimator: workload-aware | logical tokens/s: 76176.7 | valid tokens/s: 76176.7 | physical tokens/s: 77049.4 | loss tokens/s: 54680.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.985059E-01 | lm-global-supervised-tokens: 1.204283E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.817897E-02 | lm-token-buffer-mib: 2.327614E+02 | global_load_balancing_loss: 1.000385E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11160.260 | params rms: 0.060429 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:50:52.733380 | finish at 2026-10-17 18:39:00 + [2026-10-16 03:52:14] iteration 1750/ 2384 | consumed samples: 56000 | elapsed time per iteration (ms): 246587.1 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.63% | FLOPs estimator: workload-aware | logical tokens/s: 68037.5 | valid tokens/s: 68037.5 | physical tokens/s: 68672.8 | loss tokens/s: 53323.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.721751E-01 | lm-global-supervised-tokens: 1.314884E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.812264E-02 | lm-token-buffer-mib: 2.339222E+02 | global_load_balancing_loss: 1.000643E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11160.225 | params rms: 0.060429 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:25:36.220209 | finish at 2026-10-17 23:17:50 + [2026-10-16 03:55:56] iteration 1751/ 2384 | consumed samples: 56032 | elapsed time per iteration (ms): 222547.3 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.34% | FLOPs estimator: workload-aware | logical tokens/s: 75387.1 | valid tokens/s: 75387.1 | physical tokens/s: 76042.9 | loss tokens/s: 59226.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.698844E-01 | lm-global-supervised-tokens: 1.318061E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.843525E-02 | lm-token-buffer-mib: 2.349700E+02 | global_load_balancing_loss: 1.000946E+00 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11160.190 | params rms: 0.060429 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:07:52.421948 | finish at 2026-10-17 19:03:49 + [2026-10-16 03:59:39] iteration 1752/ 2384 | consumed samples: 56064 | elapsed time per iteration (ms): 222355.2 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.36% | FLOPs estimator: workload-aware | logical tokens/s: 75452.2 | valid tokens/s: 75452.2 | physical tokens/s: 76188.3 | loss tokens/s: 57088.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.862375E-01 | lm-global-supervised-tokens: 1.269393E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.780469E-02 | lm-token-buffer-mib: 2.346690E+02 | global_load_balancing_loss: 1.000838E+00 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11160.158 | params rms: 0.060429 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:02:08.465940 | finish at 2026-10-17 19:01:47 + [2026-10-16 04:03:18] iteration 1753/ 2384 | consumed samples: 56096 | elapsed time per iteration (ms): 219251.7 | throughput per GPU (TFLOP/s/GPU): 24.4 | MFU 7.81% | FLOPs estimator: workload-aware | logical tokens/s: 76520.3 | valid tokens/s: 76520.3 | physical tokens/s: 76827.7 | loss tokens/s: 56009.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.305588E-01 | lm-global-supervised-tokens: 1.228021E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.736629E-02 | lm-token-buffer-mib: 2.325584E+02 | global_load_balancing_loss: 9.995115E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11160.127 | params rms: 0.060428 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:25:47.830776 | finish at 2026-10-17 18:29:06 + [2026-10-16 04:07:01] iteration 1754/ 2384 | consumed samples: 56128 | elapsed time per iteration (ms): 222966.3 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.40% | FLOPs estimator: workload-aware | logical tokens/s: 75245.4 | valid tokens/s: 75245.3 | physical tokens/s: 75645.1 | loss tokens/s: 52034.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.263953E-01 | lm-global-supervised-tokens: 1.160200E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.731764E-02 | lm-token-buffer-mib: 2.327996E+02 | global_load_balancing_loss: 9.996848E-01 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11160.096 | params rms: 0.060428 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:01:08.797245 | finish at 2026-10-17 19:08:10 + [2026-10-16 04:10:40] iteration 1755/ 2384 | consumed samples: 56160 | elapsed time per iteration (ms): 219193.4 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.56% | FLOPs estimator: workload-aware | logical tokens/s: 76540.7 | valid tokens/s: 76540.7 | physical tokens/s: 77172.8 | loss tokens/s: 56402.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.494279E-01 | lm-global-supervised-tokens: 1.236297E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.754294E-02 | lm-token-buffer-mib: 2.334304E+02 | global_load_balancing_loss: 1.001399E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11160.063 | params rms: 0.060428 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:17:52.651390 | finish at 2026-10-17 18:28:33 + [2026-10-16 04:14:24] iteration 1756/ 2384 | consumed samples: 56192 | elapsed time per iteration (ms): 223663.2 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.51% | FLOPs estimator: workload-aware | logical tokens/s: 75011.1 | valid tokens/s: 75011.1 | physical tokens/s: 76556.0 | loss tokens/s: 56803.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.972003E-01 | lm-global-supervised-tokens: 1.270475E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.627078E-02 | lm-token-buffer-mib: 2.399167E+02 | global_load_balancing_loss: 1.000872E+00 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11160.032 | params rms: 0.060428 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:01:00.464983 | finish at 2026-10-17 19:15:24 + [2026-10-16 04:18:03] iteration 1757/ 2384 | consumed samples: 56224 | elapsed time per iteration (ms): 219799.4 | throughput per GPU (TFLOP/s/GPU): 24.6 | MFU 7.89% | FLOPs estimator: workload-aware | logical tokens/s: 76329.6 | valid tokens/s: 76329.6 | physical tokens/s: 76748.1 | loss tokens/s: 57008.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.941284E-01 | lm-global-supervised-tokens: 1.253046E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.700696E-02 | lm-token-buffer-mib: 2.324641E+02 | global_load_balancing_loss: 1.000417E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11159.999 | params rms: 0.060428 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:16:54.254652 | finish at 2026-10-17 18:34:58 + [2026-10-16 04:21:47] iteration 1758/ 2384 | consumed samples: 56256 | elapsed time per iteration (ms): 223241.1 | throughput per GPU (TFLOP/s/GPU): 19.9 | MFU 6.39% | FLOPs estimator: workload-aware | logical tokens/s: 75152.9 | valid tokens/s: 75152.9 | physical tokens/s: 76030.7 | loss tokens/s: 54024.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.922889E-01 | lm-global-supervised-tokens: 1.206058E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 6.624746E-02 | lm-token-buffer-mib: 2.345919E+02 | global_load_balancing_loss: 1.001096E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11159.967 | params rms: 0.060428 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:49:08.898049 | finish at 2026-10-17 19:10:56 + [2026-10-16 04:25:29] iteration 1759/ 2384 | consumed samples: 56288 | elapsed time per iteration (ms): 222652.6 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.86% | FLOPs estimator: workload-aware | logical tokens/s: 75351.5 | valid tokens/s: 75351.5 | physical tokens/s: 76469.7 | loss tokens/s: 59438.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.816060E-01 | lm-global-supervised-tokens: 1.323420E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.614978E-02 | lm-token-buffer-mib: 2.349044E+02 | global_load_balancing_loss: 1.000021E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11159.934 | params rms: 0.060427 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:39:17.859236 | finish at 2026-10-17 19:04:47 + [2026-10-16 04:29:41] iteration 1760/ 2384 | consumed samples: 56320 | elapsed time per iteration (ms): 251870.2 | throughput per GPU (TFLOP/s/GPU): 19.4 | MFU 6.22% | FLOPs estimator: workload-aware | logical tokens/s: 66610.5 | valid tokens/s: 66610.5 | physical tokens/s: 67479.3 | loss tokens/s: 49834.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.810386E-01 | lm-global-supervised-tokens: 1.255171E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.836742E-02 | lm-token-buffer-mib: 2.360614E+02 | global_load_balancing_loss: 1.001379E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11159.902 | params rms: 0.060427 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:39:27.028255 | finish at 2026-10-18 00:09:08 +times across ranks (ms): + save-checkpoint: + rank 0: 24906.59 + rank 1: 24906.60 + rank 2: 24906.61 + rank 3: 24906.63 + rank 4: 24906.59 + rank 5: 24906.61 + rank 6: 24906.61 + rank 7: 24906.60 + rank 8: 24906.76 + rank 9: 24906.78 + rank 10: 24906.79 + rank 11: 24906.75 + rank 12: 24906.78 + rank 13: 24906.75 + rank 14: 24906.78 + rank 15: 24906.75 + rank 16: 24906.52 + rank 17: 24906.52 + rank 18: 24906.52 + rank 19: 24906.51 + rank 20: 24906.34 + rank 21: 24906.46 + rank 22: 24906.50 + rank 23: 24906.46 + rank 24: 24906.88 + rank 25: 24906.84 + rank 26: 24906.93 + rank 27: 24906.86 + rank 28: 24906.82 + rank 29: 24907.20 + rank 30: 24906.88 + rank 31: 24906.85 + rank 32: 24907.00 + rank 33: 24907.05 + rank 34: 24907.08 + rank 35: 24907.01 + rank 36: 24907.02 + rank 37: 24907.05 + rank 38: 24906.98 + rank 39: 24906.88 + rank 40: 24907.13 + rank 41: 24907.12 + rank 42: 24907.13 + rank 43: 24907.12 + rank 44: 24907.12 + rank 45: 24907.14 + rank 46: 24907.12 + rank 47: 24907.12 + rank 48: 24906.88 + rank 49: 24906.80 + rank 50: 24906.77 + rank 51: 24906.83 + rank 52: 24906.87 + rank 53: 24906.95 + rank 54: 24906.74 + rank 55: 24906.88 + rank 56: 24906.25 + rank 57: 24906.28 + rank 58: 24906.32 + rank 59: 24906.27 + rank 60: 24906.29 + rank 61: 24906.26 + rank 62: 24906.23 + rank 63: 24906.23 + [2026-10-16 04:34:01] iteration 1761/ 2384 | consumed samples: 56352 | elapsed time per iteration (ms): 233080.1 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.72% | FLOPs estimator: workload-aware | logical tokens/s: 71980.4 | valid tokens/s: 71980.4 | physical tokens/s: 72786.9 | loss tokens/s: 53434.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.770596E-01 | lm-global-supervised-tokens: 1.245445E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.981804E-02 | lm-token-buffer-mib: 2.362395E+02 | global_load_balancing_loss: 1.000830E+00 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11159.870 | params rms: 0.060427 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:20:08.899812 | finish at 2026-10-17 20:54:10 + [2026-10-16 04:37:49] iteration 1762/ 2384 | consumed samples: 56384 | elapsed time per iteration (ms): 227497.5 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.41% | FLOPs estimator: workload-aware | logical tokens/s: 73746.8 | valid tokens/s: 73746.8 | physical tokens/s: 74674.3 | loss tokens/s: 54710.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.767282E-01 | lm-global-supervised-tokens: 1.244660E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.522990E-02 | lm-token-buffer-mib: 2.355984E+02 | global_load_balancing_loss: 1.000082E+00 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11159.839 | params rms: 0.060427 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:18:23.453566 | finish at 2026-10-17 19:56:12 + [2026-10-16 04:41:29] iteration 1763/ 2384 | consumed samples: 56416 | elapsed time per iteration (ms): 220346.9 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.37% | FLOPs estimator: workload-aware | logical tokens/s: 76139.9 | valid tokens/s: 76139.9 | physical tokens/s: 77336.4 | loss tokens/s: 57636.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.465494E-01 | lm-global-supervised-tokens: 1.270012E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.797961E-02 | lm-token-buffer-mib: 2.352076E+02 | global_load_balancing_loss: 1.001159E+00 | loss scale: 1.0 | grad norm: 0.264 | params norm: 11159.808 | params rms: 0.060427 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:00:35.413491 | finish at 2026-10-17 18:42:05 + [2026-10-16 04:45:10] iteration 1764/ 2384 | consumed samples: 56448 | elapsed time per iteration (ms): 220826.8 | throughput per GPU (TFLOP/s/GPU): 26.2 | MFU 8.40% | FLOPs estimator: workload-aware | logical tokens/s: 75974.5 | valid tokens/s: 75974.5 | physical tokens/s: 76500.9 | loss tokens/s: 57453.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.823555E-01 | lm-global-supervised-tokens: 1.268737E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.359430E-02 | lm-token-buffer-mib: 2.327021E+02 | global_load_balancing_loss: 1.000624E+00 | loss scale: 1.0 | grad norm: 0.253 | params norm: 11159.780 | params rms: 0.060427 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:01:52.619910 | finish at 2026-10-17 18:47:03 + [2026-10-16 04:48:55] iteration 1765/ 2384 | consumed samples: 56480 | elapsed time per iteration (ms): 225056.7 | throughput per GPU (TFLOP/s/GPU): 23.9 | MFU 7.65% | FLOPs estimator: workload-aware | logical tokens/s: 74546.6 | valid tokens/s: 74546.6 | physical tokens/s: 75306.8 | loss tokens/s: 52826.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.079619E-01 | lm-global-supervised-tokens: 1.188885E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.033056E-02 | lm-token-buffer-mib: 2.335401E+02 | global_load_balancing_loss: 1.000974E+00 | loss scale: 1.0 | grad norm: 0.104 | params norm: 11159.752 | params rms: 0.060426 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:41:50.069254 | finish at 2026-10-17 19:30:45 + [2026-10-16 04:52:36] iteration 1766/ 2384 | consumed samples: 56512 | elapsed time per iteration (ms): 221336.0 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.91% | FLOPs estimator: workload-aware | logical tokens/s: 75799.6 | valid tokens/s: 75799.6 | physical tokens/s: 76641.2 | loss tokens/s: 59688.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.200585E-01 | lm-global-supervised-tokens: 1.321116E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.644812E-02 | lm-token-buffer-mib: 2.331931E+02 | global_load_balancing_loss: 1.000746E+00 | loss scale: 1.0 | grad norm: 0.165 | params norm: 11159.726 | params rms: 0.060426 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:59:45.665218 | finish at 2026-10-17 18:52:22 + [2026-10-16 04:56:19] iteration 1767/ 2384 | consumed samples: 56544 | elapsed time per iteration (ms): 222913.7 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.54% | FLOPs estimator: workload-aware | logical tokens/s: 75263.2 | valid tokens/s: 75263.2 | physical tokens/s: 75887.2 | loss tokens/s: 58681.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.251661E-01 | lm-global-supervised-tokens: 1.308090E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.790982E-02 | lm-token-buffer-mib: 2.351360E+02 | global_load_balancing_loss: 1.000018E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11159.694 | params rms: 0.060426 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:12:17.747374 | finish at 2026-10-17 19:08:37 + [2026-10-16 05:00:03] iteration 1768/ 2384 | consumed samples: 56576 | elapsed time per iteration (ms): 223743.3 | throughput per GPU (TFLOP/s/GPU): 23.4 | MFU 7.51% | FLOPs estimator: workload-aware | logical tokens/s: 74984.1 | valid tokens/s: 74984.1 | physical tokens/s: 75330.9 | loss tokens/s: 54662.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.759908E-01 | lm-global-supervised-tokens: 1.223026E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.163900E-02 | lm-token-buffer-mib: 2.335345E+02 | global_load_balancing_loss: 1.000377E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11159.665 | params rms: 0.060426 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:17:05.859852 | finish at 2026-10-17 19:17:09 + [2026-10-16 05:03:46] iteration 1769/ 2384 | consumed samples: 56608 | elapsed time per iteration (ms): 223144.0 | throughput per GPU (TFLOP/s/GPU): 24.5 | MFU 7.85% | FLOPs estimator: workload-aware | logical tokens/s: 75185.5 | valid tokens/s: 75185.5 | physical tokens/s: 75451.7 | loss tokens/s: 56436.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.771938E-01 | lm-global-supervised-tokens: 1.259348E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.499096E-02 | lm-token-buffer-mib: 2.330449E+02 | global_load_balancing_loss: 1.000629E+00 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11159.636 | params rms: 0.060426 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:07:13.585546 | finish at 2026-10-17 19:11:00 + [2026-10-16 05:07:50] iteration 1770/ 2384 | consumed samples: 56640 | elapsed time per iteration (ms): 243272.9 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.37% | FLOPs estimator: workload-aware | logical tokens/s: 68964.4 | valid tokens/s: 68964.4 | physical tokens/s: 69436.8 | loss tokens/s: 53068.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.329621E-01 | lm-global-supervised-tokens: 1.291011E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.263231E-02 | lm-token-buffer-mib: 2.340332E+02 | global_load_balancing_loss: 9.998935E-01 | loss scale: 1.0 | grad norm: 0.153 | params norm: 11159.608 | params rms: 0.060426 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:29:29.544269 | finish at 2026-10-17 22:37:19 + [2026-10-16 05:11:35] iteration 1771/ 2384 | consumed samples: 56672 | elapsed time per iteration (ms): 225494.2 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.40% | FLOPs estimator: workload-aware | logical tokens/s: 74401.9 | valid tokens/s: 74401.9 | physical tokens/s: 74898.9 | loss tokens/s: 56343.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.497231E-01 | lm-global-supervised-tokens: 1.270522E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.587762E-02 | lm-token-buffer-mib: 2.341190E+02 | global_load_balancing_loss: 1.000188E+00 | loss scale: 1.0 | grad norm: 0.106 | params norm: 11159.583 | params rms: 0.060426 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:23:47.974994 | finish at 2026-10-17 19:35:23 + [2026-10-16 05:15:21] iteration 1772/ 2384 | consumed samples: 56704 | elapsed time per iteration (ms): 226417.8 | throughput per GPU (TFLOP/s/GPU): 24.6 | MFU 7.90% | FLOPs estimator: workload-aware | logical tokens/s: 74098.3 | valid tokens/s: 74098.3 | physical tokens/s: 74641.5 | loss tokens/s: 55941.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.891490E-01 | lm-global-supervised-tokens: 1.266617E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.424525E-02 | lm-token-buffer-mib: 2.341192E+02 | global_load_balancing_loss: 1.000572E+00 | loss scale: 1.0 | grad norm: 0.166 | params norm: 11159.554 | params rms: 0.060425 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:29:27.701594 | finish at 2026-10-17 19:44:49 + [2026-10-16 05:19:13] iteration 1773/ 2384 | consumed samples: 56736 | elapsed time per iteration (ms): 231326.6 | throughput per GPU (TFLOP/s/GPU): 18.5 | MFU 5.93% | FLOPs estimator: workload-aware | logical tokens/s: 72526.0 | valid tokens/s: 72526.0 | physical tokens/s: 74188.6 | loss tokens/s: 55103.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.867308E-01 | lm-global-supervised-tokens: 1.274679E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.694676E-02 | lm-token-buffer-mib: 2.392592E+02 | global_load_balancing_loss: 1.000556E+00 | loss scale: 1.0 | grad norm: 0.192 | params norm: 11159.521 | params rms: 0.060425 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:15:40.537641 | finish at 2026-10-17 20:34:53 + [2026-10-16 05:22:54] iteration 1774/ 2384 | consumed samples: 56768 | elapsed time per iteration (ms): 221489.4 | throughput per GPU (TFLOP/s/GPU): 24.6 | MFU 7.88% | FLOPs estimator: workload-aware | logical tokens/s: 75747.1 | valid tokens/s: 75747.1 | physical tokens/s: 76182.1 | loss tokens/s: 54211.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.711266E-01 | lm-global-supervised-tokens: 1.200716E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.434084E-02 | lm-token-buffer-mib: 2.343374E+02 | global_load_balancing_loss: 1.001498E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11159.491 | params rms: 0.060425 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:31:48.535254 | finish at 2026-10-17 18:54:43 + [2026-10-16 05:26:35] iteration 1775/ 2384 | consumed samples: 56800 | elapsed time per iteration (ms): 220470.8 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.26% | FLOPs estimator: workload-aware | logical tokens/s: 76097.1 | valid tokens/s: 76097.1 | physical tokens/s: 76596.2 | loss tokens/s: 56229.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.667443E-01 | lm-global-supervised-tokens: 1.239691E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.573054E-02 | lm-token-buffer-mib: 2.330948E+02 | global_load_balancing_loss: 9.997556E-01 | loss scale: 1.0 | grad norm: 0.284 | params norm: 11159.459 | params rms: 0.060425 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:17:46.734141 | finish at 2026-10-17 18:44:21 + [2026-10-16 05:30:15] iteration 1776/ 2384 | consumed samples: 56832 | elapsed time per iteration (ms): 220584.0 | throughput per GPU (TFLOP/s/GPU): 20.2 | MFU 6.49% | FLOPs estimator: workload-aware | logical tokens/s: 76058.0 | valid tokens/s: 76058.0 | physical tokens/s: 76604.5 | loss tokens/s: 55014.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.588652E-01 | lm-global-supervised-tokens: 1.213528E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.771986E-02 | lm-token-buffer-mib: 2.332832E+02 | global_load_balancing_loss: 1.000023E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11159.426 | params rms: 0.060425 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:15:15.089172 | finish at 2026-10-17 18:45:30 + [2026-10-16 05:33:58] iteration 1777/ 2384 | consumed samples: 56864 | elapsed time per iteration (ms): 222720.3 | throughput per GPU (TFLOP/s/GPU): 23.5 | MFU 7.55% | FLOPs estimator: workload-aware | logical tokens/s: 75328.5 | valid tokens/s: 75328.5 | physical tokens/s: 75950.8 | loss tokens/s: 56356.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.175919E-01 | lm-global-supervised-tokens: 1.255167E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.451174E-02 | lm-token-buffer-mib: 2.343119E+02 | global_load_balancing_loss: 9.994726E-01 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11159.393 | params rms: 0.060424 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:33:11.229601 | finish at 2026-10-17 19:07:09 + [2026-10-16 05:37:40] iteration 1778/ 2384 | consumed samples: 56896 | elapsed time per iteration (ms): 222173.4 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.47% | FLOPs estimator: workload-aware | logical tokens/s: 75513.8 | valid tokens/s: 75513.8 | physical tokens/s: 76123.0 | loss tokens/s: 57328.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.930365E-01 | lm-global-supervised-tokens: 1.273693E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.762814E-02 | lm-token-buffer-mib: 2.341957E+02 | global_load_balancing_loss: 9.999455E-01 | loss scale: 1.0 | grad norm: 0.358 | params norm: 11159.360 | params rms: 0.060424 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:23:57.069952 | finish at 2026-10-17 19:01:37 + [2026-10-16 05:41:20] iteration 1779/ 2384 | consumed samples: 56928 | elapsed time per iteration (ms): 219841.1 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.34% | FLOPs estimator: workload-aware | logical tokens/s: 76315.2 | valid tokens/s: 76315.2 | physical tokens/s: 77558.9 | loss tokens/s: 59991.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.224168E-01 | lm-global-supervised-tokens: 1.318859E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.289661E-02 | lm-token-buffer-mib: 2.343242E+02 | global_load_balancing_loss: 9.999493E-01 | loss scale: 1.0 | grad norm: 0.317 | params norm: 11159.330 | params rms: 0.060424 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:56:43.839955 | finish at 2026-10-17 18:38:04 + [2026-10-16 05:45:23] iteration 1780/ 2384 | consumed samples: 56960 | elapsed time per iteration (ms): 243255.4 | throughput per GPU (TFLOP/s/GPU): 19.5 | MFU 6.26% | FLOPs estimator: workload-aware | logical tokens/s: 68969.6 | valid tokens/s: 68969.6 | physical tokens/s: 69781.7 | loss tokens/s: 49954.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.796114E-01 | lm-global-supervised-tokens: 1.215162E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.545360E-02 | lm-token-buffer-mib: 2.367484E+02 | global_load_balancing_loss: 1.000538E+00 | loss scale: 1.0 | grad norm: 0.119 | params norm: 11159.303 | params rms: 0.060424 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:48:46.236796 | finish at 2026-10-17 22:34:10 + [2026-10-16 05:49:07] iteration 1781/ 2384 | consumed samples: 56992 | elapsed time per iteration (ms): 224068.3 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.32% | FLOPs estimator: workload-aware | logical tokens/s: 74875.3 | valid tokens/s: 74875.3 | physical tokens/s: 75572.1 | loss tokens/s: 60516.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.153746E-01 | lm-global-supervised-tokens: 1.355989E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.382738E-02 | lm-token-buffer-mib: 2.336788E+02 | global_load_balancing_loss: 1.001354E+00 | loss scale: 1.0 | grad norm: 0.095 | params norm: 11159.276 | params rms: 0.060424 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:31:53.192237 | finish at 2026-10-17 19:21:01 + [2026-10-16 05:52:48] iteration 1782/ 2384 | consumed samples: 57024 | elapsed time per iteration (ms): 220661.5 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.71% | FLOPs estimator: workload-aware | logical tokens/s: 76031.4 | valid tokens/s: 76031.4 | physical tokens/s: 76929.1 | loss tokens/s: 54031.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.792645E-01 | lm-global-supervised-tokens: 1.192257E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.804980E-02 | lm-token-buffer-mib: 2.360217E+02 | global_load_balancing_loss: 1.001569E+00 | loss scale: 1.0 | grad norm: 0.112 | params norm: 11159.248 | params rms: 0.060424 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:53:58.234612 | finish at 2026-10-17 18:46:46 + [2026-10-16 05:56:33] iteration 1783/ 2384 | consumed samples: 57056 | elapsed time per iteration (ms): 225026.2 | throughput per GPU (TFLOP/s/GPU): 23.8 | MFU 7.63% | FLOPs estimator: workload-aware | logical tokens/s: 74556.7 | valid tokens/s: 74556.7 | physical tokens/s: 75321.5 | loss tokens/s: 54947.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.551754E-01 | lm-global-supervised-tokens: 1.236465E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.358968E-02 | lm-token-buffer-mib: 2.335970E+02 | global_load_balancing_loss: 1.001930E+00 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11159.219 | params rms: 0.060424 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:34:00.751965 | finish at 2026-10-17 19:30:34 + [2026-10-16 06:00:19] iteration 1784/ 2384 | consumed samples: 57088 | elapsed time per iteration (ms): 226229.5 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.19% | FLOPs estimator: workload-aware | logical tokens/s: 74160.0 | valid tokens/s: 74160.0 | physical tokens/s: 74638.6 | loss tokens/s: 53235.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.050870E-01 | lm-global-supervised-tokens: 1.204348E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.952971E-02 | lm-token-buffer-mib: 2.329543E+02 | global_load_balancing_loss: 1.000838E+00 | loss scale: 1.0 | grad norm: 0.111 | params norm: 11159.190 | params rms: 0.060423 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:42:17.674284 | finish at 2026-10-17 19:42:37 + [2026-10-16 06:04:04] iteration 1785/ 2384 | consumed samples: 57120 | elapsed time per iteration (ms): 225063.8 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.84% | FLOPs estimator: workload-aware | logical tokens/s: 74544.0 | valid tokens/s: 74544.0 | physical tokens/s: 75098.2 | loss tokens/s: 56959.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.969489E-01 | lm-global-supervised-tokens: 1.281959E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.734008E-02 | lm-token-buffer-mib: 2.335003E+02 | global_load_balancing_loss: 1.000629E+00 | loss scale: 1.0 | grad norm: 0.033 | params norm: 11159.160 | params rms: 0.060423 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:26:53.223398 | finish at 2026-10-17 19:30:58 + [2026-10-16 06:07:51] iteration 1786/ 2384 | consumed samples: 57152 | elapsed time per iteration (ms): 226583.4 | throughput per GPU (TFLOP/s/GPU): 25.3 | MFU 8.11% | FLOPs estimator: workload-aware | logical tokens/s: 74044.1 | valid tokens/s: 74044.1 | physical tokens/s: 74255.6 | loss tokens/s: 56614.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.115286E-01 | lm-global-supervised-tokens: 1.282795E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.344609E-02 | lm-token-buffer-mib: 2.330335E+02 | global_load_balancing_loss: 1.000938E+00 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11159.129 | params rms: 0.060423 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:38:16.870070 | finish at 2026-10-17 19:46:08 + [2026-10-16 06:11:33] iteration 1787/ 2384 | consumed samples: 57184 | elapsed time per iteration (ms): 222233.1 | throughput per GPU (TFLOP/s/GPU): 23.2 | MFU 7.45% | FLOPs estimator: workload-aware | logical tokens/s: 75493.7 | valid tokens/s: 75493.7 | physical tokens/s: 76290.1 | loss tokens/s: 51502.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.786922E-01 | lm-global-supervised-tokens: 1.144561E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.894261E-02 | lm-token-buffer-mib: 2.347183E+02 | global_load_balancing_loss: 9.991002E-01 | loss scale: 1.0 | grad norm: 0.125 | params norm: 11159.093 | params rms: 0.060423 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:51:13.156250 | finish at 2026-10-17 19:02:46 + [2026-10-16 06:15:18] iteration 1788/ 2384 | consumed samples: 57216 | elapsed time per iteration (ms): 225151.8 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.15% | FLOPs estimator: workload-aware | logical tokens/s: 74515.1 | valid tokens/s: 74515.1 | physical tokens/s: 75384.4 | loss tokens/s: 56257.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.059970E-01 | lm-global-supervised-tokens: 1.266646E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.907740E-02 | lm-token-buffer-mib: 2.346846E+02 | global_load_balancing_loss: 9.985527E-01 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11159.056 | params rms: 0.060423 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:16:30.488666 | finish at 2026-10-17 19:31:49 + [2026-10-16 06:18:59] iteration 1789/ 2384 | consumed samples: 57248 | elapsed time per iteration (ms): 220476.5 | throughput per GPU (TFLOP/s/GPU): 24.2 | MFU 7.77% | FLOPs estimator: workload-aware | logical tokens/s: 76094.9 | valid tokens/s: 76094.9 | physical tokens/s: 76541.7 | loss tokens/s: 57556.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.519226E-01 | lm-global-supervised-tokens: 1.268994E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.515376E-02 | lm-token-buffer-mib: 2.331395E+02 | global_load_balancing_loss: 9.985766E-01 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11159.020 | params rms: 0.060422 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:26:23.535111 | finish at 2026-10-17 18:45:22 + [2026-10-16 06:23:02] iteration 1790/ 2384 | consumed samples: 57280 | elapsed time per iteration (ms): 243046.7 | throughput per GPU (TFLOP/s/GPU): 18.7 | MFU 5.99% | FLOPs estimator: workload-aware | logical tokens/s: 69028.7 | valid tokens/s: 69028.7 | physical tokens/s: 69547.8 | loss tokens/s: 50985.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.648144E-01 | lm-global-supervised-tokens: 1.239189E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.060932E-02 | lm-token-buffer-mib: 2.328455E+02 | global_load_balancing_loss: 1.000088E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11158.982 | params rms: 0.060422 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:06:09.714592 | finish at 2026-10-17 22:29:12 + [2026-10-16 06:26:45] iteration 1791/ 2384 | consumed samples: 57312 | elapsed time per iteration (ms): 223197.0 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.31% | FLOPs estimator: workload-aware | logical tokens/s: 75167.7 | valid tokens/s: 75167.7 | physical tokens/s: 76151.9 | loss tokens/s: 60834.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.682294E-01 | lm-global-supervised-tokens: 1.357812E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.701396E-02 | lm-token-buffer-mib: 2.362854E+02 | global_load_balancing_loss: 1.000804E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11158.945 | params rms: 0.060422 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:45:55.811260 | finish at 2026-10-17 19:12:41 + [2026-10-16 06:30:28] iteration 1792/ 2384 | consumed samples: 57344 | elapsed time per iteration (ms): 223083.2 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.38% | FLOPs estimator: workload-aware | logical tokens/s: 75205.9 | valid tokens/s: 75205.9 | physical tokens/s: 76189.0 | loss tokens/s: 54679.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.841559E-01 | lm-global-supervised-tokens: 1.219805E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.751603E-02 | lm-token-buffer-mib: 2.348583E+02 | global_load_balancing_loss: 1.002621E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11158.909 | params rms: 0.060422 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:41:05.244083 | finish at 2026-10-17 19:11:33 + [2026-10-16 06:34:10] iteration 1793/ 2384 | consumed samples: 57376 | elapsed time per iteration (ms): 221652.5 | throughput per GPU (TFLOP/s/GPU): 25.5 | MFU 8.18% | FLOPs estimator: workload-aware | logical tokens/s: 75691.2 | valid tokens/s: 75691.2 | physical tokens/s: 76746.3 | loss tokens/s: 53718.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.093213E-01 | lm-global-supervised-tokens: 1.190680E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.696758E-02 | lm-token-buffer-mib: 2.333361E+02 | global_load_balancing_loss: 1.002305E+00 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11158.877 | params rms: 0.060422 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:23:16.598141 | finish at 2026-10-17 18:57:26 + [2026-10-16 06:37:49] iteration 1794/ 2384 | consumed samples: 57408 | elapsed time per iteration (ms): 219039.6 | throughput per GPU (TFLOP/s/GPU): 19.6 | MFU 6.28% | FLOPs estimator: workload-aware | logical tokens/s: 76594.4 | valid tokens/s: 76594.4 | physical tokens/s: 77243.1 | loss tokens/s: 57729.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.605546E-01 | lm-global-supervised-tokens: 1.264507E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.504330E-02 | lm-token-buffer-mib: 2.325859E+02 | global_load_balancing_loss: 1.001263E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11158.843 | params rms: 0.060422 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:53:53.363657 | finish at 2026-10-17 18:31:42 + [2026-10-16 06:41:35] iteration 1795/ 2384 | consumed samples: 57440 | elapsed time per iteration (ms): 225889.9 | throughput per GPU (TFLOP/s/GPU): 23.8 | MFU 7.62% | FLOPs estimator: workload-aware | logical tokens/s: 74271.5 | valid tokens/s: 74271.5 | physical tokens/s: 75286.5 | loss tokens/s: 57274.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.139663E-01 | lm-global-supervised-tokens: 1.293782E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.490287E-02 | lm-token-buffer-mib: 2.376234E+02 | global_load_balancing_loss: 9.998565E-01 | loss scale: 1.0 | grad norm: 0.173 | params norm: 11158.807 | params rms: 0.060421 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:57:29.132967 | finish at 2026-10-17 19:39:04 + [2026-10-16 06:45:18] iteration 1796/ 2384 | consumed samples: 57472 | elapsed time per iteration (ms): 223126.7 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.04% | FLOPs estimator: workload-aware | logical tokens/s: 75191.3 | valid tokens/s: 75191.3 | physical tokens/s: 76057.1 | loss tokens/s: 54078.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.597667E-01 | lm-global-supervised-tokens: 1.206630E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.676264E-02 | lm-token-buffer-mib: 2.365973E+02 | global_load_balancing_loss: 9.998260E-01 | loss scale: 1.0 | grad norm: 0.160 | params norm: 11158.772 | params rms: 0.060421 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:26:38.509789 | finish at 2026-10-17 19:11:56 + [2026-10-16 06:49:01] iteration 1797/ 2384 | consumed samples: 57504 | elapsed time per iteration (ms): 222963.4 | throughput per GPU (TFLOP/s/GPU): 24.3 | MFU 7.79% | FLOPs estimator: workload-aware | logical tokens/s: 75246.2 | valid tokens/s: 75246.2 | physical tokens/s: 76306.3 | loss tokens/s: 60951.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.797194E-01 | lm-global-supervised-tokens: 1.359005E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.293620E-02 | lm-token-buffer-mib: 2.348149E+02 | global_load_balancing_loss: 9.990296E-01 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11158.736 | params rms: 0.060421 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:21:19.524131 | finish at 2026-10-17 19:10:20 + [2026-10-16 06:52:46] iteration 1798/ 2384 | consumed samples: 57536 | elapsed time per iteration (ms): 225562.4 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.86% | FLOPs estimator: workload-aware | logical tokens/s: 74379.2 | valid tokens/s: 74379.2 | physical tokens/s: 75155.2 | loss tokens/s: 63346.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.068435E-01 | lm-global-supervised-tokens: 1.428853E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.807042E-02 | lm-token-buffer-mib: 2.343357E+02 | global_load_balancing_loss: 9.990770E-01 | loss scale: 1.0 | grad norm: 0.185 | params norm: 11158.701 | params rms: 0.060421 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:42:59.575262 | finish at 2026-10-17 19:35:46 + [2026-10-16 06:56:29] iteration 1799/ 2384 | consumed samples: 57568 | elapsed time per iteration (ms): 222413.2 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.46% | FLOPs estimator: workload-aware | logical tokens/s: 75432.5 | valid tokens/s: 75432.5 | physical tokens/s: 75717.5 | loss tokens/s: 55786.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.106880E-01 | lm-global-supervised-tokens: 1.240765E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.469165E-02 | lm-token-buffer-mib: 2.327170E+02 | global_load_balancing_loss: 1.000145E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11158.664 | params rms: 0.060421 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:08:31.745653 | finish at 2026-10-17 19:05:01 + [2026-10-16 07:00:31] iteration 1800/ 2384 | consumed samples: 57600 | elapsed time per iteration (ms): 241922.5 | throughput per GPU (TFLOP/s/GPU): 17.5 | MFU 5.61% | FLOPs estimator: workload-aware | logical tokens/s: 69349.5 | valid tokens/s: 69349.5 | physical tokens/s: 70067.0 | loss tokens/s: 52177.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.603688E-01 | lm-global-supervised-tokens: 1.262289E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.338200E-02 | lm-token-buffer-mib: 2.341575E+02 | global_load_balancing_loss: 1.000421E+00 | loss scale: 1.0 | grad norm: 0.137 | params norm: 11158.626 | params rms: 0.060420 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:14:42.730192 | finish at 2026-10-17 22:15:13 +times across ranks (ms): + save-checkpoint: + rank 0: 25389.29 + rank 1: 25389.32 + rank 2: 25389.17 + rank 3: 25389.29 + rank 4: 25389.25 + rank 5: 25389.20 + rank 6: 25389.18 + rank 7: 25389.17 + rank 8: 25389.36 + rank 9: 25389.39 + rank 10: 25389.40 + rank 11: 25389.33 + rank 12: 25389.34 + rank 13: 25389.34 + rank 14: 25389.36 + rank 15: 25389.35 + rank 16: 25388.57 + rank 17: 25388.56 + rank 18: 25388.57 + rank 19: 25388.58 + rank 20: 25388.54 + rank 21: 25388.54 + rank 22: 25388.58 + rank 23: 25388.55 + rank 24: 25390.14 + rank 25: 25390.12 + rank 26: 25390.15 + rank 27: 25390.10 + rank 28: 25389.85 + rank 29: 25390.13 + rank 30: 25390.13 + rank 31: 25390.12 + rank 32: 25388.54 + rank 33: 25388.69 + rank 34: 25388.59 + rank 35: 25388.55 + rank 36: 25388.60 + rank 37: 25388.57 + rank 38: 25388.57 + rank 39: 25388.59 + rank 40: 25388.65 + rank 41: 25388.64 + rank 42: 25388.62 + rank 43: 25388.64 + rank 44: 25388.60 + rank 45: 25388.61 + rank 46: 25388.61 + rank 47: 25388.61 + rank 48: 25387.26 + rank 49: 25387.30 + rank 50: 25387.45 + rank 51: 25387.42 + rank 52: 25387.38 + rank 53: 25387.45 + rank 54: 25387.26 + rank 55: 25387.37 + rank 56: 25388.73 + rank 57: 25388.77 + rank 58: 25388.77 + rank 59: 25388.75 + rank 60: 25388.76 + rank 61: 25388.73 + rank 62: 25388.73 + rank 63: 25388.73 + [2026-10-16 07:04:39] iteration 1801/ 2384 | consumed samples: 57632 | elapsed time per iteration (ms): 222542.7 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.40% | FLOPs estimator: workload-aware | logical tokens/s: 75388.6 | valid tokens/s: 75388.6 | physical tokens/s: 76349.6 | loss tokens/s: 57759.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.053053E-01 | lm-global-supervised-tokens: 1.285390E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.735566E-02 | lm-token-buffer-mib: 2.357122E+02 | global_load_balancing_loss: 1.001113E+00 | loss scale: 1.0 | grad norm: 0.105 | params norm: 11158.584 | params rms: 0.060420 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:02:22.400802 | finish at 2026-10-17 19:07:01 + [2026-10-16 07:08:31] iteration 1802/ 2384 | consumed samples: 57664 | elapsed time per iteration (ms): 231782.6 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.74% | FLOPs estimator: workload-aware | logical tokens/s: 72383.3 | valid tokens/s: 72383.3 | physical tokens/s: 73541.5 | loss tokens/s: 53112.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.064953E-01 | lm-global-supervised-tokens: 1.231049E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.749474E-02 | lm-token-buffer-mib: 2.363189E+02 | global_load_balancing_loss: 1.001667E+00 | loss scale: 1.0 | grad norm: 0.133 | params norm: 11158.544 | params rms: 0.060420 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:28:17.497440 | finish at 2026-10-17 20:36:48 + [2026-10-16 07:12:14] iteration 1803/ 2384 | consumed samples: 57696 | elapsed time per iteration (ms): 223336.4 | throughput per GPU (TFLOP/s/GPU): 24.0 | MFU 7.70% | FLOPs estimator: workload-aware | logical tokens/s: 75120.8 | valid tokens/s: 75120.8 | physical tokens/s: 75502.8 | loss tokens/s: 56661.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.299214E-01 | lm-global-supervised-tokens: 1.265460E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.489656E-02 | lm-token-buffer-mib: 2.335343E+02 | global_load_balancing_loss: 1.001500E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11158.502 | params rms: 0.060420 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:02:38.476400 | finish at 2026-10-17 19:14:52 + [2026-10-16 07:15:55] iteration 1804/ 2384 | consumed samples: 57728 | elapsed time per iteration (ms): 221532.0 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.91% | FLOPs estimator: workload-aware | logical tokens/s: 75732.6 | valid tokens/s: 75732.6 | physical tokens/s: 76148.1 | loss tokens/s: 57572.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.818285E-01 | lm-global-supervised-tokens: 1.275407E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.769783E-02 | lm-token-buffer-mib: 2.336543E+02 | global_load_balancing_loss: 1.001094E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11158.458 | params rms: 0.060419 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:41:28.563771 | finish at 2026-10-17 18:57:24 + [2026-10-16 07:19:40] iteration 1805/ 2384 | consumed samples: 57760 | elapsed time per iteration (ms): 224123.3 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.39% | FLOPs estimator: workload-aware | logical tokens/s: 74856.9 | valid tokens/s: 74856.9 | physical tokens/s: 75585.0 | loss tokens/s: 57940.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.291259E-01 | lm-global-supervised-tokens: 1.298580E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.140080E-02 | lm-token-buffer-mib: 2.329133E+02 | global_load_balancing_loss: 1.000162E+00 | loss scale: 1.0 | grad norm: 0.285 | params norm: 11158.414 | params rms: 0.060419 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:02:47.379424 | finish at 2026-10-17 19:22:27 + [2026-10-16 07:23:27] iteration 1806/ 2384 | consumed samples: 57792 | elapsed time per iteration (ms): 227374.6 | throughput per GPU (TFLOP/s/GPU): 24.0 | MFU 7.70% | FLOPs estimator: workload-aware | logical tokens/s: 73786.6 | valid tokens/s: 73786.6 | physical tokens/s: 74601.5 | loss tokens/s: 50414.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.639579E-01 | lm-global-supervised-tokens: 1.146291E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.258167E-02 | lm-token-buffer-mib: 2.340376E+02 | global_load_balancing_loss: 9.995502E-01 | loss scale: 1.0 | grad norm: 0.155 | params norm: 11158.368 | params rms: 0.060419 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:30:22.530751 | finish at 2026-10-17 19:53:49 + [2026-10-16 07:27:11] iteration 1807/ 2384 | consumed samples: 57824 | elapsed time per iteration (ms): 224360.3 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.40% | FLOPs estimator: workload-aware | logical tokens/s: 74777.9 | valid tokens/s: 74777.9 | physical tokens/s: 75162.2 | loss tokens/s: 55921.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.840348E-01 | lm-global-supervised-tokens: 1.254647E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.888240E-02 | lm-token-buffer-mib: 2.332901E+02 | global_load_balancing_loss: 9.987783E-01 | loss scale: 1.0 | grad norm: 0.338 | params norm: 11158.320 | params rms: 0.060419 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:57:35.893137 | finish at 2026-10-17 19:24:47 + [2026-10-16 07:30:55] iteration 1808/ 2384 | consumed samples: 57856 | elapsed time per iteration (ms): 224150.7 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.14% | FLOPs estimator: workload-aware | logical tokens/s: 74847.9 | valid tokens/s: 74847.9 | physical tokens/s: 75327.0 | loss tokens/s: 54892.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.985058E-01 | lm-global-supervised-tokens: 1.230421E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.383635E-02 | lm-token-buffer-mib: 2.341836E+02 | global_load_balancing_loss: 9.986558E-01 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11158.277 | params rms: 0.060418 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:51:50.782379 | finish at 2026-10-17 19:22:46 + [2026-10-16 07:34:42] iteration 1809/ 2384 | consumed samples: 57888 | elapsed time per iteration (ms): 226081.0 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.36% | FLOPs estimator: workload-aware | logical tokens/s: 74208.7 | valid tokens/s: 74208.7 | physical tokens/s: 74574.6 | loss tokens/s: 53458.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.803657E-01 | lm-global-supervised-tokens: 1.208584E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.330822E-02 | lm-token-buffer-mib: 2.336986E+02 | global_load_balancing_loss: 1.000546E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11158.239 | params rms: 0.060418 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:06:36.584374 | finish at 2026-10-17 19:41:18 + [2026-10-16 07:38:48] iteration 1810/ 2384 | consumed samples: 57920 | elapsed time per iteration (ms): 246275.0 | throughput per GPU (TFLOP/s/GPU): 23.5 | MFU 7.52% | FLOPs estimator: workload-aware | logical tokens/s: 68123.8 | valid tokens/s: 68123.8 | physical tokens/s: 68773.9 | loss tokens/s: 49178.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.466571E-01 | lm-global-supervised-tokens: 1.211144E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.840727E-02 | lm-token-buffer-mib: 2.355885E+02 | global_load_balancing_loss: 1.000549E+00 | loss scale: 1.0 | grad norm: 0.210 | params norm: 11158.206 | params rms: 0.060418 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:16:01.849781 | finish at 2026-10-17 22:54:50 + [2026-10-16 07:42:30] iteration 1811/ 2384 | consumed samples: 57952 | elapsed time per iteration (ms): 222052.4 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.03% | FLOPs estimator: workload-aware | logical tokens/s: 75555.1 | valid tokens/s: 75555.1 | physical tokens/s: 76254.2 | loss tokens/s: 57245.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.055134E-01 | lm-global-supervised-tokens: 1.271153E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.753390E-02 | lm-token-buffer-mib: 2.332409E+02 | global_load_balancing_loss: 9.999173E-01 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11158.173 | params rms: 0.060418 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:20:36.013789 | finish at 2026-10-17 19:03:06 + [2026-10-16 07:46:15] iteration 1812/ 2384 | consumed samples: 57984 | elapsed time per iteration (ms): 225385.2 | throughput per GPU (TFLOP/s/GPU): 24.5 | MFU 7.85% | FLOPs estimator: workload-aware | logical tokens/s: 74437.8 | valid tokens/s: 74437.8 | physical tokens/s: 75801.0 | loss tokens/s: 60832.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.825119E-01 | lm-global-supervised-tokens: 1.371083E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.834394E-02 | lm-token-buffer-mib: 2.347272E+02 | global_load_balancing_loss: 1.000520E+00 | loss scale: 1.0 | grad norm: 0.146 | params norm: 11158.140 | params rms: 0.060418 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:48:40.319549 | finish at 2026-10-17 19:34:56 + [2026-10-16 07:50:00] iteration 1813/ 2384 | consumed samples: 58016 | elapsed time per iteration (ms): 224714.5 | throughput per GPU (TFLOP/s/GPU): 19.1 | MFU 6.13% | FLOPs estimator: workload-aware | logical tokens/s: 74660.1 | valid tokens/s: 74660.1 | physical tokens/s: 77024.0 | loss tokens/s: 60185.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.894625E-01 | lm-global-supervised-tokens: 1.352459E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.839310E-02 | lm-token-buffer-mib: 2.368250E+02 | global_load_balancing_loss: 1.001042E+00 | loss scale: 1.0 | grad norm: 0.194 | params norm: 11158.107 | params rms: 0.060418 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:38:31.964224 | finish at 2026-10-17 19:28:32 + [2026-10-16 07:53:44] iteration 1814/ 2384 | consumed samples: 58048 | elapsed time per iteration (ms): 223844.9 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.95% | FLOPs estimator: workload-aware | logical tokens/s: 74950.0 | valid tokens/s: 74950.0 | physical tokens/s: 75713.3 | loss tokens/s: 57854.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.649906E-01 | lm-global-supervised-tokens: 1.295052E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.327357E-02 | lm-token-buffer-mib: 2.353134E+02 | global_load_balancing_loss: 1.001453E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11158.081 | params rms: 0.060417 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:26:31.583834 | finish at 2026-10-17 19:20:15 + [2026-10-16 07:57:27] iteration 1815/ 2384 | consumed samples: 58080 | elapsed time per iteration (ms): 223458.5 | throughput per GPU (TFLOP/s/GPU): 19.4 | MFU 6.21% | FLOPs estimator: workload-aware | logical tokens/s: 75079.6 | valid tokens/s: 75079.6 | physical tokens/s: 76026.1 | loss tokens/s: 56094.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.525883E-01 | lm-global-supervised-tokens: 1.253484E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.334703E-02 | lm-token-buffer-mib: 2.335166E+02 | global_load_balancing_loss: 1.001656E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11158.058 | params rms: 0.060417 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:19:07.869897 | finish at 2026-10-17 19:16:35 + [2026-10-16 08:01:10] iteration 1816/ 2384 | consumed samples: 58112 | elapsed time per iteration (ms): 223028.2 | throughput per GPU (TFLOP/s/GPU): 19.0 | MFU 6.09% | FLOPs estimator: workload-aware | logical tokens/s: 75224.7 | valid tokens/s: 75224.6 | physical tokens/s: 76292.7 | loss tokens/s: 56753.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.547425E-01 | lm-global-supervised-tokens: 1.265766E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.528392E-02 | lm-token-buffer-mib: 2.351588E+02 | global_load_balancing_loss: 1.000387E+00 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11158.031 | params rms: 0.060417 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:11:19.991278 | finish at 2026-10-17 19:12:30 + [2026-10-16 08:04:52] iteration 1817/ 2384 | consumed samples: 58144 | elapsed time per iteration (ms): 221815.5 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.63% | FLOPs estimator: workload-aware | logical tokens/s: 75635.7 | valid tokens/s: 75635.7 | physical tokens/s: 76880.9 | loss tokens/s: 59321.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.509022E-01 | lm-global-supervised-tokens: 1.315845E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.899957E-02 | lm-token-buffer-mib: 2.358708E+02 | global_load_balancing_loss: 1.000699E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11158.005 | params rms: 0.060417 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:56:09.363773 | finish at 2026-10-17 19:01:01 + [2026-10-16 08:08:36] iteration 1818/ 2384 | consumed samples: 58176 | elapsed time per iteration (ms): 223704.1 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.40% | FLOPs estimator: workload-aware | logical tokens/s: 74997.4 | valid tokens/s: 74997.4 | physical tokens/s: 76203.8 | loss tokens/s: 55303.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.257870E-01 | lm-global-supervised-tokens: 1.237168E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.310181E-02 | lm-token-buffer-mib: 2.368946E+02 | global_load_balancing_loss: 9.997388E-01 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11157.978 | params rms: 0.060417 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:10:16.526882 | finish at 2026-10-17 19:18:52 + [2026-10-16 08:12:18] iteration 1819/ 2384 | consumed samples: 58208 | elapsed time per iteration (ms): 222323.5 | throughput per GPU (TFLOP/s/GPU): 25.4 | MFU 8.14% | FLOPs estimator: workload-aware | logical tokens/s: 75463.0 | valid tokens/s: 75463.0 | physical tokens/s: 75732.5 | loss tokens/s: 53708.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.880811E-01 | lm-global-supervised-tokens: 1.194061E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.993056E-02 | lm-token-buffer-mib: 2.330541E+02 | global_load_balancing_loss: 1.000615E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11157.951 | params rms: 0.060417 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:53:32.794023 | finish at 2026-10-17 19:05:51 + [2026-10-16 08:16:21] iteration 1820/ 2384 | consumed samples: 58240 | elapsed time per iteration (ms): 243083.9 | throughput per GPU (TFLOP/s/GPU): 17.7 | MFU 5.68% | FLOPs estimator: workload-aware | logical tokens/s: 69018.2 | valid tokens/s: 69018.2 | physical tokens/s: 70180.1 | loss tokens/s: 54417.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.274380E-01 | lm-global-supervised-tokens: 1.322804E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.740591E-02 | lm-token-buffer-mib: 2.359658E+02 | global_load_balancing_loss: 9.993958E-01 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11157.924 | params rms: 0.060417 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:04:59.310173 | finish at 2026-10-17 22:21:21 + [2026-10-16 08:20:08] iteration 1821/ 2384 | consumed samples: 58272 | elapsed time per iteration (ms): 226943.4 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.98% | FLOPs estimator: workload-aware | logical tokens/s: 73926.8 | valid tokens/s: 73926.8 | physical tokens/s: 74993.1 | loss tokens/s: 54805.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.397429E-01 | lm-global-supervised-tokens: 1.243768E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.350987E-02 | lm-token-buffer-mib: 2.381002E+02 | global_load_balancing_loss: 9.992098E-01 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11157.894 | params rms: 0.060416 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:29:29.143812 | finish at 2026-10-17 19:49:37 + [2026-10-16 08:23:50] iteration 1822/ 2384 | consumed samples: 58304 | elapsed time per iteration (ms): 221872.7 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.98% | FLOPs estimator: workload-aware | logical tokens/s: 75616.3 | valid tokens/s: 75616.3 | physical tokens/s: 76256.8 | loss tokens/s: 58473.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.977738E-01 | lm-global-supervised-tokens: 1.297369E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.869935E-02 | lm-token-buffer-mib: 2.352827E+02 | global_load_balancing_loss: 1.001130E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11157.864 | params rms: 0.060416 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:38:12.458994 | finish at 2026-10-17 19:02:02 + [2026-10-16 08:27:33] iteration 1823/ 2384 | consumed samples: 58336 | elapsed time per iteration (ms): 222528.1 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.17% | FLOPs estimator: workload-aware | logical tokens/s: 75393.6 | valid tokens/s: 75393.6 | physical tokens/s: 76258.5 | loss tokens/s: 62765.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.788747E-01 | lm-global-supervised-tokens: 1.396707E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.885098E-02 | lm-token-buffer-mib: 2.349898E+02 | global_load_balancing_loss: 1.000523E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11157.833 | params rms: 0.060416 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:40:38.260496 | finish at 2026-10-17 19:08:11 + [2026-10-16 08:31:16] iteration 1824/ 2384 | consumed samples: 58368 | elapsed time per iteration (ms): 223640.6 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.32% | FLOPs estimator: workload-aware | logical tokens/s: 75018.6 | valid tokens/s: 75018.6 | physical tokens/s: 75553.8 | loss tokens/s: 54859.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.187051E-01 | lm-global-supervised-tokens: 1.226885E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.353452E-02 | lm-token-buffer-mib: 2.334401E+02 | global_load_balancing_loss: 1.003178E+00 | loss scale: 1.0 | grad norm: 0.148 | params norm: 11157.804 | params rms: 0.060416 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:47:18.726368 | finish at 2026-10-17 19:18:35 + [2026-10-16 08:34:59] iteration 1825/ 2384 | consumed samples: 58400 | elapsed time per iteration (ms): 222417.6 | throughput per GPU (TFLOP/s/GPU): 19.0 | MFU 6.08% | FLOPs estimator: workload-aware | logical tokens/s: 75431.1 | valid tokens/s: 75431.1 | physical tokens/s: 76053.0 | loss tokens/s: 55447.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.143096E-01 | lm-global-supervised-tokens: 1.233239E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.959020E-02 | lm-token-buffer-mib: 2.335726E+02 | global_load_balancing_loss: 1.001437E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11157.773 | params rms: 0.060416 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:32:11.456346 | finish at 2026-10-17 19:07:10 + [2026-10-16 08:38:42] iteration 1826/ 2384 | consumed samples: 58432 | elapsed time per iteration (ms): 223725.9 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.78% | FLOPs estimator: workload-aware | logical tokens/s: 74989.9 | valid tokens/s: 74989.9 | physical tokens/s: 76134.9 | loss tokens/s: 56676.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.644354E-01 | lm-global-supervised-tokens: 1.267992E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.432907E-02 | lm-token-buffer-mib: 2.356425E+02 | global_load_balancing_loss: 1.000155E+00 | loss scale: 1.0 | grad norm: 0.508 | params norm: 11157.744 | params rms: 0.060416 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:40:39.027552 | finish at 2026-10-17 19:19:21 + [2026-10-16 08:42:26] iteration 1827/ 2384 | consumed samples: 58464 | elapsed time per iteration (ms): 223184.8 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.19% | FLOPs estimator: workload-aware | logical tokens/s: 75171.8 | valid tokens/s: 75171.8 | physical tokens/s: 75709.2 | loss tokens/s: 52662.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.018597E-01 | lm-global-supervised-tokens: 1.175356E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.084621E-02 | lm-token-buffer-mib: 2.326871E+02 | global_load_balancing_loss: 1.000460E+00 | loss scale: 1.0 | grad norm: 0.173 | params norm: 11157.715 | params rms: 0.060415 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:31:53.926777 | finish at 2026-10-17 19:14:19 + [2026-10-16 08:46:06] iteration 1828/ 2384 | consumed samples: 58496 | elapsed time per iteration (ms): 220970.9 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.11% | FLOPs estimator: workload-aware | logical tokens/s: 75924.8 | valid tokens/s: 75924.8 | physical tokens/s: 76570.0 | loss tokens/s: 54488.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.499922E-01 | lm-global-supervised-tokens: 1.204030E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.986013E-02 | lm-token-buffer-mib: 2.339456E+02 | global_load_balancing_loss: 9.994376E-01 | loss scale: 1.0 | grad norm: 0.375 | params norm: 11157.684 | params rms: 0.060415 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:07:39.823216 | finish at 2026-10-17 18:53:46 + [2026-10-16 08:49:50] iteration 1829/ 2384 | consumed samples: 58528 | elapsed time per iteration (ms): 223058.6 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.79% | FLOPs estimator: workload-aware | logical tokens/s: 75214.4 | valid tokens/s: 75214.3 | physical tokens/s: 75704.2 | loss tokens/s: 52720.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.132885E-01 | lm-global-supervised-tokens: 1.175970E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.889731E-02 | lm-token-buffer-mib: 2.342433E+02 | global_load_balancing_loss: 9.984314E-01 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11157.653 | params rms: 0.060415 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:23:17.497036 | finish at 2026-10-17 19:13:07 + [2026-10-16 08:53:57] iteration 1830/ 2384 | consumed samples: 58560 | elapsed time per iteration (ms): 247359.3 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.35% | FLOPs estimator: workload-aware | logical tokens/s: 67825.2 | valid tokens/s: 67825.2 | physical tokens/s: 68345.6 | loss tokens/s: 51019.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.227042E-01 | lm-global-supervised-tokens: 1.262020E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.149157E-02 | lm-token-buffer-mib: 2.343871E+02 | global_load_balancing_loss: 9.990314E-01 | loss scale: 1.0 | grad norm: 0.243 | params norm: 11157.623 | params rms: 0.060415 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:03:57.056502 | finish at 2026-10-17 22:57:54 + [2026-10-16 08:57:38] iteration 1831/ 2384 | consumed samples: 58592 | elapsed time per iteration (ms): 221512.4 | throughput per GPU (TFLOP/s/GPU): 25.7 | MFU 8.24% | FLOPs estimator: workload-aware | logical tokens/s: 75739.2 | valid tokens/s: 75739.2 | physical tokens/s: 76657.6 | loss tokens/s: 60525.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.712974E-01 | lm-global-supervised-tokens: 1.340703E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.354048E-02 | lm-token-buffer-mib: 2.335373E+02 | global_load_balancing_loss: 1.000979E+00 | loss scale: 1.0 | grad norm: 0.198 | params norm: 11157.591 | params rms: 0.060415 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:01:36.371654 | finish at 2026-10-17 18:59:15 + [2026-10-16 09:01:19] iteration 1832/ 2384 | consumed samples: 58624 | elapsed time per iteration (ms): 220469.7 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.12% | FLOPs estimator: workload-aware | logical tokens/s: 76097.5 | valid tokens/s: 76097.5 | physical tokens/s: 76428.8 | loss tokens/s: 54803.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.888142E-01 | lm-global-supervised-tokens: 1.208256E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.637781E-02 | lm-token-buffer-mib: 2.324161E+02 | global_load_balancing_loss: 1.001651E+00 | loss scale: 1.0 | grad norm: 0.123 | params norm: 11157.557 | params rms: 0.060415 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:48:19.252607 | finish at 2026-10-17 18:49:38 + [2026-10-16 09:05:04] iteration 1833/ 2384 | consumed samples: 58656 | elapsed time per iteration (ms): 224723.2 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.87% | FLOPs estimator: workload-aware | logical tokens/s: 74657.2 | valid tokens/s: 74657.2 | physical tokens/s: 75741.8 | loss tokens/s: 57910.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.100664E-01 | lm-global-supervised-tokens: 1.301390E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.873377E-02 | lm-token-buffer-mib: 2.344669E+02 | global_load_balancing_loss: 1.001756E+00 | loss scale: 1.0 | grad norm: 0.183 | params norm: 11157.524 | params rms: 0.060414 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:23:42.463146 | finish at 2026-10-17 19:28:46 + [2026-10-16 09:08:46] iteration 1834/ 2384 | consumed samples: 58688 | elapsed time per iteration (ms): 222561.4 | throughput per GPU (TFLOP/s/GPU): 23.4 | MFU 7.50% | FLOPs estimator: workload-aware | logical tokens/s: 75382.4 | valid tokens/s: 75382.4 | physical tokens/s: 76255.5 | loss tokens/s: 59496.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.670310E-01 | lm-global-supervised-tokens: 1.324167E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.187831E-02 | lm-token-buffer-mib: 2.359622E+02 | global_load_balancing_loss: 1.001379E+00 | loss scale: 1.0 | grad norm: 0.184 | params norm: 11157.491 | params rms: 0.060414 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:00:08.750820 | finish at 2026-10-17 19:08:55 + [2026-10-16 09:12:31] iteration 1835/ 2384 | consumed samples: 58720 | elapsed time per iteration (ms): 224436.1 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.82% | FLOPs estimator: workload-aware | logical tokens/s: 74752.6 | valid tokens/s: 74752.6 | physical tokens/s: 75602.0 | loss tokens/s: 59178.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.192533E-01 | lm-global-supervised-tokens: 1.328180E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.702061E-02 | lm-token-buffer-mib: 2.342811E+02 | global_load_balancing_loss: 1.000435E+00 | loss scale: 1.0 | grad norm: 0.133 | params norm: 11157.457 | params rms: 0.060414 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:13:35.423746 | finish at 2026-10-17 19:26:06 + [2026-10-16 09:16:10] iteration 1836/ 2384 | consumed samples: 58752 | elapsed time per iteration (ms): 219868.6 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.17% | FLOPs estimator: workload-aware | logical tokens/s: 76305.6 | valid tokens/s: 76305.6 | physical tokens/s: 77038.6 | loss tokens/s: 59972.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.602989E-01 | lm-global-supervised-tokens: 1.318608E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.387275E-02 | lm-token-buffer-mib: 2.331925E+02 | global_load_balancing_loss: 9.998122E-01 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11157.421 | params rms: 0.060414 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:28:07.980590 | finish at 2026-10-17 18:44:18 + [2026-10-16 09:19:54] iteration 1837/ 2384 | consumed samples: 58784 | elapsed time per iteration (ms): 223649.2 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.57% | FLOPs estimator: workload-aware | logical tokens/s: 75015.6 | valid tokens/s: 75015.6 | physical tokens/s: 75551.2 | loss tokens/s: 55736.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.229550E-01 | lm-global-supervised-tokens: 1.246544E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.174759E-02 | lm-token-buffer-mib: 2.332361E+02 | global_load_balancing_loss: 9.994258E-01 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11157.388 | params rms: 0.060414 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:58:56.115249 | finish at 2026-10-17 19:18:50 + [2026-10-16 09:23:37] iteration 1838/ 2384 | consumed samples: 58816 | elapsed time per iteration (ms): 223338.3 | throughput per GPU (TFLOP/s/GPU): 19.7 | MFU 6.32% | FLOPs estimator: workload-aware | logical tokens/s: 75120.0 | valid tokens/s: 75120.0 | physical tokens/s: 75855.5 | loss tokens/s: 55677.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.302218E-01 | lm-global-supervised-tokens: 1.243489E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.871239E-02 | lm-token-buffer-mib: 2.342387E+02 | global_load_balancing_loss: 9.998024E-01 | loss scale: 1.0 | grad norm: 0.115 | params norm: 11157.359 | params rms: 0.060413 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:52:22.713096 | finish at 2026-10-17 19:16:00 + [2026-10-16 09:27:22] iteration 1839/ 2384 | consumed samples: 58848 | elapsed time per iteration (ms): 224714.0 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.73% | FLOPs estimator: workload-aware | logical tokens/s: 74660.0 | valid tokens/s: 74660.0 | physical tokens/s: 75493.6 | loss tokens/s: 58666.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.324567E-01 | lm-global-supervised-tokens: 1.318311E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.678088E-02 | lm-token-buffer-mib: 2.358339E+02 | global_load_balancing_loss: 9.997333E-01 | loss scale: 1.0 | grad norm: 0.165 | params norm: 11157.329 | params rms: 0.060413 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:01:09.149613 | finish at 2026-10-17 19:28:31 + [2026-10-16 09:31:28] iteration 1840/ 2384 | consumed samples: 58880 | elapsed time per iteration (ms): 245652.8 | throughput per GPU (TFLOP/s/GPU): 20.2 | MFU 6.49% | FLOPs estimator: workload-aware | logical tokens/s: 68296.3 | valid tokens/s: 68296.3 | physical tokens/s: 68986.4 | loss tokens/s: 52319.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.443852E-01 | lm-global-supervised-tokens: 1.285237E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.148386E-02 | lm-token-buffer-mib: 2.350104E+02 | global_load_balancing_loss: 9.991933E-01 | loss scale: 1.0 | grad norm: 0.121 | params norm: 11157.297 | params rms: 0.060413 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:07:15.102364 | finish at 2026-10-17 22:38:43 +times across ranks (ms): + save-checkpoint: + rank 0: 25029.96 + rank 1: 25029.95 + rank 2: 25030.04 + rank 3: 25029.97 + rank 4: 25029.96 + rank 5: 25029.99 + rank 6: 25029.97 + rank 7: 25029.98 + rank 8: 25029.69 + rank 9: 25029.65 + rank 10: 25029.70 + rank 11: 25029.66 + rank 12: 25029.67 + rank 13: 25029.77 + rank 14: 25029.67 + rank 15: 25029.69 + rank 16: 25030.30 + rank 17: 25030.29 + rank 18: 25030.34 + rank 19: 25030.37 + rank 20: 25030.25 + rank 21: 25030.26 + rank 22: 25030.29 + rank 23: 25030.25 + rank 24: 25029.73 + rank 25: 25029.73 + rank 26: 25029.79 + rank 27: 25029.71 + rank 28: 25029.72 + rank 29: 25029.76 + rank 30: 25029.69 + rank 31: 25029.98 + rank 32: 25030.12 + rank 33: 25030.14 + rank 34: 25030.12 + rank 35: 25030.15 + rank 36: 25030.17 + rank 37: 25030.12 + rank 38: 25030.08 + rank 39: 25030.15 + rank 40: 25029.71 + rank 41: 25029.69 + rank 42: 25029.68 + rank 43: 25029.69 + rank 44: 25029.66 + rank 45: 25029.70 + rank 46: 25029.71 + rank 47: 25029.65 + rank 48: 25029.85 + rank 49: 25029.87 + rank 50: 25029.89 + rank 51: 25029.82 + rank 52: 25029.83 + rank 53: 25029.84 + rank 54: 25029.85 + rank 55: 25029.81 + rank 56: 25030.50 + rank 57: 25030.48 + rank 58: 25030.51 + rank 59: 25030.38 + rank 60: 25030.57 + rank 61: 25030.47 + rank 62: 25030.23 + rank 63: 25030.53 + [2026-10-16 09:35:41] iteration 1841/ 2384 | consumed samples: 58912 | elapsed time per iteration (ms): 228369.9 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.50% | FLOPs estimator: workload-aware | logical tokens/s: 73465.0 | valid tokens/s: 73465.0 | physical tokens/s: 74166.0 | loss tokens/s: 54856.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.648835E-01 | lm-global-supervised-tokens: 1.252763E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.534974E-02 | lm-token-buffer-mib: 2.344732E+02 | global_load_balancing_loss: 1.000474E+00 | loss scale: 1.0 | grad norm: 0.150 | params norm: 11157.265 | params rms: 0.060413 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:26:44.852328 | finish at 2026-10-17 20:02:26 + [2026-10-16 09:39:29] iteration 1842/ 2384 | consumed samples: 58944 | elapsed time per iteration (ms): 227404.1 | throughput per GPU (TFLOP/s/GPU): 23.2 | MFU 7.45% | FLOPs estimator: workload-aware | logical tokens/s: 73777.0 | valid tokens/s: 73777.0 | physical tokens/s: 74407.5 | loss tokens/s: 58641.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.052635E-01 | lm-global-supervised-tokens: 1.333532E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.392507E-02 | lm-token-buffer-mib: 2.335406E+02 | global_load_balancing_loss: 1.001464E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11157.231 | params rms: 0.060413 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:14:13.045041 | finish at 2026-10-17 19:53:42 + [2026-10-16 09:43:14] iteration 1843/ 2384 | consumed samples: 58976 | elapsed time per iteration (ms): 224887.6 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.75% | FLOPs estimator: workload-aware | logical tokens/s: 74602.4 | valid tokens/s: 74602.4 | physical tokens/s: 75541.8 | loss tokens/s: 54056.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.305421E-01 | lm-global-supervised-tokens: 1.215655E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.545964E-02 | lm-token-buffer-mib: 2.361125E+02 | global_load_balancing_loss: 1.001780E+00 | loss scale: 1.0 | grad norm: 0.307 | params norm: 11157.196 | params rms: 0.060413 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:47:44.217496 | finish at 2026-10-17 19:30:58 + [2026-10-16 09:46:55] iteration 1844/ 2384 | consumed samples: 59008 | elapsed time per iteration (ms): 221589.3 | throughput per GPU (TFLOP/s/GPU): 24.3 | MFU 7.79% | FLOPs estimator: workload-aware | logical tokens/s: 75712.9 | valid tokens/s: 75712.9 | physical tokens/s: 76535.1 | loss tokens/s: 55957.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.729006E-01 | lm-global-supervised-tokens: 1.239953E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.994652E-02 | lm-token-buffer-mib: 2.353939E+02 | global_load_balancing_loss: 1.000229E+00 | loss scale: 1.0 | grad norm: 0.236 | params norm: 11157.162 | params rms: 0.060412 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:14:18.240623 | finish at 2026-10-17 19:01:14 + [2026-10-16 09:50:34] iteration 1845/ 2384 | consumed samples: 59040 | elapsed time per iteration (ms): 218906.4 | throughput per GPU (TFLOP/s/GPU): 25.1 | MFU 8.06% | FLOPs estimator: workload-aware | logical tokens/s: 76640.9 | valid tokens/s: 76640.9 | physical tokens/s: 77122.3 | loss tokens/s: 54771.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.906090E-01 | lm-global-supervised-tokens: 1.198972E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.704438E-02 | lm-token-buffer-mib: 2.337941E+02 | global_load_balancing_loss: 1.000973E+00 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11157.127 | params rms: 0.060412 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:46:30.523366 | finish at 2026-10-17 18:37:05 + [2026-10-16 09:54:18] iteration 1846/ 2384 | consumed samples: 59072 | elapsed time per iteration (ms): 223611.5 | throughput per GPU (TFLOP/s/GPU): 19.7 | MFU 6.31% | FLOPs estimator: workload-aware | logical tokens/s: 75028.4 | valid tokens/s: 75028.4 | physical tokens/s: 75827.2 | loss tokens/s: 55164.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.551230E-01 | lm-global-supervised-tokens: 1.233542E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.424349E-02 | lm-token-buffer-mib: 2.338699E+02 | global_load_balancing_loss: 1.000145E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11157.095 | params rms: 0.060412 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:25:02.963540 | finish at 2026-10-17 19:19:21 + [2026-10-16 09:57:59] iteration 1847/ 2384 | consumed samples: 59104 | elapsed time per iteration (ms): 221454.3 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.83% | FLOPs estimator: workload-aware | logical tokens/s: 75759.1 | valid tokens/s: 75759.1 | physical tokens/s: 76495.6 | loss tokens/s: 55978.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.038835E-01 | lm-global-supervised-tokens: 1.239660E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.803540E-02 | lm-token-buffer-mib: 2.345125E+02 | global_load_balancing_loss: 9.994586E-01 | loss scale: 1.0 | grad norm: 0.233 | params norm: 11157.059 | params rms: 0.060412 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:02:00.948434 | finish at 2026-10-17 19:00:00 + [2026-10-16 10:01:40] iteration 1848/ 2384 | consumed samples: 59136 | elapsed time per iteration (ms): 220869.7 | throughput per GPU (TFLOP/s/GPU): 26.1 | MFU 8.35% | FLOPs estimator: workload-aware | logical tokens/s: 75959.7 | valid tokens/s: 75959.7 | physical tokens/s: 76355.6 | loss tokens/s: 56023.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.850871E-01 | lm-global-supervised-tokens: 1.237388E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.291261E-02 | lm-token-buffer-mib: 2.331988E+02 | global_load_balancing_loss: 9.999731E-01 | loss scale: 1.0 | grad norm: 0.032 | params norm: 11157.023 | params rms: 0.060412 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:53:06.177067 | finish at 2026-10-17 18:54:46 + [2026-10-16 10:05:21] iteration 1849/ 2384 | consumed samples: 59168 | elapsed time per iteration (ms): 221239.9 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.99% | FLOPs estimator: workload-aware | logical tokens/s: 75832.6 | valid tokens/s: 75832.6 | physical tokens/s: 76443.9 | loss tokens/s: 58094.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.211293E-01 | lm-global-supervised-tokens: 1.285276E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.310125E-02 | lm-token-buffer-mib: 2.330860E+02 | global_load_balancing_loss: 9.992919E-01 | loss scale: 1.0 | grad norm: 0.116 | params norm: 11156.985 | params rms: 0.060411 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:52:43.346688 | finish at 2026-10-17 18:58:05 + [2026-10-16 10:09:24] iteration 1850/ 2384 | consumed samples: 59200 | elapsed time per iteration (ms): 242814.6 | throughput per GPU (TFLOP/s/GPU): 18.4 | MFU 5.91% | FLOPs estimator: workload-aware | logical tokens/s: 69094.7 | valid tokens/s: 69094.7 | physical tokens/s: 69741.0 | loss tokens/s: 56227.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.744374E-01 | lm-global-supervised-tokens: 1.365277E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.091119E-02 | lm-token-buffer-mib: 2.342262E+02 | global_load_balancing_loss: 1.000279E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11156.947 | params rms: 0.060411 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:01:02.987738 | finish at 2026-10-17 22:10:27 + [2026-10-16 10:13:11] iteration 1851/ 2384 | consumed samples: 59232 | elapsed time per iteration (ms): 226369.3 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.58% | FLOPs estimator: workload-aware | logical tokens/s: 74114.3 | valid tokens/s: 74114.3 | physical tokens/s: 75483.9 | loss tokens/s: 54830.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.371049E-01 | lm-global-supervised-tokens: 1.241200E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.412222E-02 | lm-token-buffer-mib: 2.385817E+02 | global_load_balancing_loss: 1.001006E+00 | loss scale: 1.0 | grad norm: 0.128 | params norm: 11156.911 | params rms: 0.060411 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:30:54.854250 | finish at 2026-10-17 19:44:06 + [2026-10-16 10:16:54] iteration 1852/ 2384 | consumed samples: 59264 | elapsed time per iteration (ms): 223227.8 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.36% | FLOPs estimator: workload-aware | logical tokens/s: 75157.4 | valid tokens/s: 75157.4 | physical tokens/s: 75637.6 | loss tokens/s: 60829.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.907415E-01 | lm-global-supervised-tokens: 1.357887E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.252840E-02 | lm-token-buffer-mib: 2.332849E+02 | global_load_balancing_loss: 1.000803E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11156.874 | params rms: 0.060411 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:59:17.175083 | finish at 2026-10-17 19:16:11 + [2026-10-16 10:20:35] iteration 1853/ 2384 | consumed samples: 59296 | elapsed time per iteration (ms): 221554.5 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 7.00% | FLOPs estimator: workload-aware | logical tokens/s: 75724.9 | valid tokens/s: 75724.9 | physical tokens/s: 76845.2 | loss tokens/s: 59123.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.611331E-01 | lm-global-supervised-tokens: 1.309917E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.521993E-02 | lm-token-buffer-mib: 2.345241E+02 | global_load_balancing_loss: 1.000540E+00 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11156.840 | params rms: 0.060411 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:40:45.465381 | finish at 2026-10-17 19:01:21 + [2026-10-16 10:24:16] iteration 1854/ 2384 | consumed samples: 59328 | elapsed time per iteration (ms): 220908.7 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.79% | FLOPs estimator: workload-aware | logical tokens/s: 75946.3 | valid tokens/s: 75946.3 | physical tokens/s: 76779.3 | loss tokens/s: 60972.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.444265E-01 | lm-global-supervised-tokens: 1.346924E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.656771E-02 | lm-token-buffer-mib: 2.334124E+02 | global_load_balancing_loss: 1.001020E+00 | loss scale: 1.0 | grad norm: 0.118 | params norm: 11156.804 | params rms: 0.060410 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:31:21.605182 | finish at 2026-10-17 18:55:38 + [2026-10-16 10:27:58] iteration 1855/ 2384 | consumed samples: 59360 | elapsed time per iteration (ms): 221508.9 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.01% | FLOPs estimator: workload-aware | logical tokens/s: 75740.5 | valid tokens/s: 75740.5 | physical tokens/s: 76577.9 | loss tokens/s: 58621.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.943190E-01 | lm-global-supervised-tokens: 1.298521E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.593557E-02 | lm-token-buffer-mib: 2.342348E+02 | global_load_balancing_loss: 1.000116E+00 | loss scale: 1.0 | grad norm: 0.291 | params norm: 11156.765 | params rms: 0.060410 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:32:58.199359 | finish at 2026-10-17 19:00:56 + [2026-10-16 10:31:43] iteration 1856/ 2384 | consumed samples: 59392 | elapsed time per iteration (ms): 224949.0 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.68% | FLOPs estimator: workload-aware | logical tokens/s: 74582.1 | valid tokens/s: 74582.1 | physical tokens/s: 75056.3 | loss tokens/s: 60171.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.217186E-01 | lm-global-supervised-tokens: 1.353558E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.173219E-02 | lm-token-buffer-mib: 2.329725E+02 | global_load_balancing_loss: 1.000411E+00 | loss scale: 1.0 | grad norm: 0.294 | params norm: 11156.727 | params rms: 0.060410 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:59:33.084148 | finish at 2026-10-17 19:31:16 + [2026-10-16 10:35:24] iteration 1857/ 2384 | consumed samples: 59424 | elapsed time per iteration (ms): 221694.0 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.47% | FLOPs estimator: workload-aware | logical tokens/s: 75677.3 | valid tokens/s: 75677.3 | physical tokens/s: 76570.6 | loss tokens/s: 59925.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.788886E-01 | lm-global-supervised-tokens: 1.328505E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.092464E-02 | lm-token-buffer-mib: 2.333222E+02 | global_load_balancing_loss: 9.990840E-01 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11156.692 | params rms: 0.060410 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:27:12.718151 | finish at 2026-10-17 19:02:37 + [2026-10-16 10:39:09] iteration 1858/ 2384 | consumed samples: 59456 | elapsed time per iteration (ms): 224316.2 | throughput per GPU (TFLOP/s/GPU): 24.6 | MFU 7.88% | FLOPs estimator: workload-aware | logical tokens/s: 74792.5 | valid tokens/s: 74792.5 | physical tokens/s: 75622.7 | loss tokens/s: 54569.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.484378E-01 | lm-global-supervised-tokens: 1.224074E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.252181E-02 | lm-token-buffer-mib: 2.336405E+02 | global_load_balancing_loss: 9.999619E-01 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11156.654 | params rms: 0.060410 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:46:30.325975 | finish at 2026-10-17 19:25:39 + [2026-10-16 10:42:49] iteration 1859/ 2384 | consumed samples: 59488 | elapsed time per iteration (ms): 219973.4 | throughput per GPU (TFLOP/s/GPU): 23.2 | MFU 7.43% | FLOPs estimator: workload-aware | logical tokens/s: 76269.2 | valid tokens/s: 76269.2 | physical tokens/s: 76942.2 | loss tokens/s: 57896.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.927900E-01 | lm-global-supervised-tokens: 1.273570E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.233616E-02 | lm-token-buffer-mib: 2.339248E+02 | global_load_balancing_loss: 9.996926E-01 | loss scale: 1.0 | grad norm: 0.189 | params norm: 11156.617 | params rms: 0.060409 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:04:46.044449 | finish at 2026-10-17 18:47:35 + [2026-10-16 10:46:52] iteration 1860/ 2384 | consumed samples: 59520 | elapsed time per iteration (ms): 243347.1 | throughput per GPU (TFLOP/s/GPU): 23.9 | MFU 7.66% | FLOPs estimator: workload-aware | logical tokens/s: 68943.4 | valid tokens/s: 68943.4 | physical tokens/s: 69506.1 | loss tokens/s: 51263.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.128107E-01 | lm-global-supervised-tokens: 1.247490E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.609226E-02 | lm-token-buffer-mib: 2.343008E+02 | global_load_balancing_loss: 1.001627E+00 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11156.580 | params rms: 0.060409 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:25:13.860296 | finish at 2026-10-17 22:12:06 + [2026-10-16 10:50:37] iteration 1861/ 2384 | consumed samples: 59552 | elapsed time per iteration (ms): 225271.5 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.88% | FLOPs estimator: workload-aware | logical tokens/s: 74475.3 | valid tokens/s: 74475.3 | physical tokens/s: 75677.6 | loss tokens/s: 58470.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.652899E-01 | lm-global-supervised-tokens: 1.317176E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.598177E-02 | lm-token-buffer-mib: 2.363957E+02 | global_load_balancing_loss: 1.001413E+00 | loss scale: 1.0 | grad norm: 0.272 | params norm: 11156.542 | params rms: 0.060409 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:43:36.974981 | finish at 2026-10-17 19:34:14 + [2026-10-16 10:54:18] iteration 1862/ 2384 | consumed samples: 59584 | elapsed time per iteration (ms): 220824.2 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.89% | FLOPs estimator: workload-aware | logical tokens/s: 75975.2 | valid tokens/s: 75975.2 | physical tokens/s: 76403.8 | loss tokens/s: 56622.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.831258E-01 | lm-global-supervised-tokens: 1.250352E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.293537E-02 | lm-token-buffer-mib: 2.337512E+02 | global_load_balancing_loss: 1.001103E+00 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11156.506 | params rms: 0.060409 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:01:10.249779 | finish at 2026-10-17 18:55:28 + [2026-10-16 10:58:01] iteration 1863/ 2384 | consumed samples: 59616 | elapsed time per iteration (ms): 222913.9 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.42% | FLOPs estimator: workload-aware | logical tokens/s: 75263.1 | valid tokens/s: 75263.1 | physical tokens/s: 75797.8 | loss tokens/s: 51405.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.060967E-01 | lm-global-supervised-tokens: 1.145908E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.503069E-02 | lm-token-buffer-mib: 2.341762E+02 | global_load_balancing_loss: 1.001025E+00 | loss scale: 1.0 | grad norm: 0.258 | params norm: 11156.471 | params rms: 0.060409 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:15:38.164703 | finish at 2026-10-17 19:13:39 + [2026-10-16 11:01:42] iteration 1864/ 2384 | consumed samples: 59648 | elapsed time per iteration (ms): 221349.1 | throughput per GPU (TFLOP/s/GPU): 18.3 | MFU 5.88% | FLOPs estimator: workload-aware | logical tokens/s: 75795.2 | valid tokens/s: 75795.2 | physical tokens/s: 76801.2 | loss tokens/s: 57594.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.741635E-01 | lm-global-supervised-tokens: 1.274855E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.172698E-02 | lm-token-buffer-mib: 2.336312E+02 | global_load_balancing_loss: 9.995048E-01 | loss scale: 1.0 | grad norm: 0.301 | params norm: 11156.438 | params rms: 0.060408 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:58:21.509743 | finish at 2026-10-17 19:00:04 + [2026-10-16 11:05:24] iteration 1865/ 2384 | consumed samples: 59680 | elapsed time per iteration (ms): 221355.5 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.54% | FLOPs estimator: workload-aware | logical tokens/s: 75793.0 | valid tokens/s: 75793.0 | physical tokens/s: 77201.1 | loss tokens/s: 58039.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.889143E-01 | lm-global-supervised-tokens: 1.284739E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.322927E-02 | lm-token-buffer-mib: 2.340764E+02 | global_load_balancing_loss: 9.992508E-01 | loss scale: 1.0 | grad norm: 0.230 | params norm: 11156.406 | params rms: 0.060408 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:54:43.508822 | finish at 2026-10-17 19:00:07 + [2026-10-16 11:09:10] iteration 1866/ 2384 | consumed samples: 59712 | elapsed time per iteration (ms): 225747.9 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.17% | FLOPs estimator: workload-aware | logical tokens/s: 74318.3 | valid tokens/s: 74318.3 | physical tokens/s: 75076.7 | loss tokens/s: 54707.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.307037E-01 | lm-global-supervised-tokens: 1.235007E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.955289E-02 | lm-token-buffer-mib: 2.352199E+02 | global_load_balancing_loss: 9.993988E-01 | loss scale: 1.0 | grad norm: 0.265 | params norm: 11156.375 | params rms: 0.060408 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:28:57.400225 | finish at 2026-10-17 19:38:07 + [2026-10-16 11:12:51] iteration 1867/ 2384 | consumed samples: 59744 | elapsed time per iteration (ms): 221674.1 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.34% | FLOPs estimator: workload-aware | logical tokens/s: 75684.0 | valid tokens/s: 75683.9 | physical tokens/s: 76412.9 | loss tokens/s: 57156.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.555119E-01 | lm-global-supervised-tokens: 1.267001E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.068924E-02 | lm-token-buffer-mib: 2.349969E+02 | global_load_balancing_loss: 1.000204E+00 | loss scale: 1.0 | grad norm: 0.196 | params norm: 11156.345 | params rms: 0.060408 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:50:05.522233 | finish at 2026-10-17 19:02:57 + [2026-10-16 11:16:36] iteration 1868/ 2384 | consumed samples: 59776 | elapsed time per iteration (ms): 225076.5 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.15% | FLOPs estimator: workload-aware | logical tokens/s: 74539.8 | valid tokens/s: 74539.8 | physical tokens/s: 75832.4 | loss tokens/s: 61958.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.756235E-01 | lm-global-supervised-tokens: 1.394549E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.427016E-02 | lm-token-buffer-mib: 2.379537E+02 | global_load_balancing_loss: 1.000891E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11156.315 | params rms: 0.060408 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:15:39.482826 | finish at 2026-10-17 19:32:16 + [2026-10-16 11:20:18] iteration 1869/ 2384 | consumed samples: 59808 | elapsed time per iteration (ms): 221605.6 | throughput per GPU (TFLOP/s/GPU): 20.2 | MFU 6.47% | FLOPs estimator: workload-aware | logical tokens/s: 75707.5 | valid tokens/s: 75707.5 | physical tokens/s: 76363.4 | loss tokens/s: 55314.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.199038E-01 | lm-global-supervised-tokens: 1.225799E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.233722E-02 | lm-token-buffer-mib: 2.337784E+02 | global_load_balancing_loss: 1.000188E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11156.282 | params rms: 0.060408 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:42:06.880009 | finish at 2026-10-17 19:02:25 + [2026-10-16 11:24:26] iteration 1870/ 2384 | consumed samples: 59840 | elapsed time per iteration (ms): 248128.1 | throughput per GPU (TFLOP/s/GPU): 19.0 | MFU 6.09% | FLOPs estimator: workload-aware | logical tokens/s: 67615.0 | valid tokens/s: 67615.0 | physical tokens/s: 68857.3 | loss tokens/s: 53847.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.359414E-01 | lm-global-supervised-tokens: 1.336116E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.823666E-02 | lm-token-buffer-mib: 2.357551E+02 | global_load_balancing_loss: 1.001793E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11156.252 | params rms: 0.060407 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:25:37.837721 | finish at 2026-10-17 22:50:04 + [2026-10-16 11:28:11] iteration 1871/ 2384 | consumed samples: 59872 | elapsed time per iteration (ms): 224825.5 | throughput per GPU (TFLOP/s/GPU): 25.2 | MFU 8.07% | FLOPs estimator: workload-aware | logical tokens/s: 74623.1 | valid tokens/s: 74623.1 | physical tokens/s: 74789.8 | loss tokens/s: 56367.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.877275E-01 | lm-global-supervised-tokens: 1.267286E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.507755E-02 | lm-token-buffer-mib: 2.325131E+02 | global_load_balancing_loss: 1.000761E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11156.221 | params rms: 0.060407 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:02:15.497284 | finish at 2026-10-17 19:30:26 + [2026-10-16 11:31:52] iteration 1872/ 2384 | consumed samples: 59904 | elapsed time per iteration (ms): 221407.2 | throughput per GPU (TFLOP/s/GPU): 23.4 | MFU 7.49% | FLOPs estimator: workload-aware | logical tokens/s: 75775.2 | valid tokens/s: 75775.2 | physical tokens/s: 76103.7 | loss tokens/s: 52896.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.468046E-01 | lm-global-supervised-tokens: 1.171167E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.407037E-02 | lm-token-buffer-mib: 2.326440E+02 | global_load_balancing_loss: 1.000135E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11156.190 | params rms: 0.060407 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:29:20.503296 | finish at 2026-10-17 19:01:13 + [2026-10-16 11:35:32] iteration 1873/ 2384 | consumed samples: 59936 | elapsed time per iteration (ms): 219603.7 | throughput per GPU (TFLOP/s/GPU): 23.8 | MFU 7.62% | FLOPs estimator: workload-aware | logical tokens/s: 76397.6 | valid tokens/s: 76397.6 | physical tokens/s: 76999.8 | loss tokens/s: 53111.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.782350E-01 | lm-global-supervised-tokens: 1.166337E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.542556E-02 | lm-token-buffer-mib: 2.342599E+02 | global_load_balancing_loss: 1.001066E+00 | loss scale: 1.0 | grad norm: 0.214 | params norm: 11156.159 | params rms: 0.060407 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:10:17.486396 | finish at 2026-10-17 18:45:49 + [2026-10-16 11:39:12] iteration 1874/ 2384 | consumed samples: 59968 | elapsed time per iteration (ms): 220296.7 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.70% | FLOPs estimator: workload-aware | logical tokens/s: 76157.3 | valid tokens/s: 76157.3 | physical tokens/s: 76545.2 | loss tokens/s: 58406.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.236513E-01 | lm-global-supervised-tokens: 1.286675E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.323757E-02 | lm-token-buffer-mib: 2.335601E+02 | global_load_balancing_loss: 1.000605E+00 | loss scale: 1.0 | grad norm: 0.133 | params norm: 11156.124 | params rms: 0.060407 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:12:31.329038 | finish at 2026-10-17 18:51:44 + [2026-10-16 11:42:57] iteration 1875/ 2384 | consumed samples: 60000 | elapsed time per iteration (ms): 224908.1 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.90% | FLOPs estimator: workload-aware | logical tokens/s: 74595.8 | valid tokens/s: 74595.8 | physical tokens/s: 75254.6 | loss tokens/s: 58557.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.438836E-01 | lm-global-supervised-tokens: 1.317000E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.680231E-02 | lm-token-buffer-mib: 2.346082E+02 | global_load_balancing_loss: 1.000024E+00 | loss scale: 1.0 | grad norm: 0.174 | params norm: 11156.091 | params rms: 0.060407 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:47:58.226242 | finish at 2026-10-17 19:30:55 + [2026-10-16 11:46:39] iteration 1876/ 2384 | consumed samples: 60032 | elapsed time per iteration (ms): 221475.5 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.59% | FLOPs estimator: workload-aware | logical tokens/s: 75752.0 | valid tokens/s: 75752.0 | physical tokens/s: 76360.6 | loss tokens/s: 52629.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.292490E-01 | lm-global-supervised-tokens: 1.165611E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.584062E-02 | lm-token-buffer-mib: 2.342663E+02 | global_load_balancing_loss: 9.998960E-01 | loss scale: 1.0 | grad norm: 0.100 | params norm: 11156.060 | params rms: 0.060406 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:15:09.547877 | finish at 2026-10-17 19:01:48 + [2026-10-16 11:50:25] iteration 1877/ 2384 | consumed samples: 60064 | elapsed time per iteration (ms): 226451.6 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.35% | FLOPs estimator: workload-aware | logical tokens/s: 74087.4 | valid tokens/s: 74087.4 | physical tokens/s: 75063.3 | loss tokens/s: 54692.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.145229E-01 | lm-global-supervised-tokens: 1.238518E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.514899E-02 | lm-token-buffer-mib: 2.367019E+02 | global_load_balancing_loss: 9.992715E-01 | loss scale: 1.0 | grad norm: 0.276 | params norm: 11156.031 | params rms: 0.060406 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:53:30.942623 | finish at 2026-10-17 19:43:56 + [2026-10-16 11:54:08] iteration 1878/ 2384 | consumed samples: 60096 | elapsed time per iteration (ms): 223066.3 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.57% | FLOPs estimator: workload-aware | logical tokens/s: 75211.7 | valid tokens/s: 75211.7 | physical tokens/s: 75777.9 | loss tokens/s: 55722.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.954510E-01 | lm-global-supervised-tokens: 1.242985E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.529195E-02 | lm-token-buffer-mib: 2.334133E+02 | global_load_balancing_loss: 9.993212E-01 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11156.005 | params rms: 0.060406 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:21:11.552583 | finish at 2026-10-17 19:15:20 + [2026-10-16 11:57:54] iteration 1879/ 2384 | consumed samples: 60128 | elapsed time per iteration (ms): 225471.1 | throughput per GPU (TFLOP/s/GPU): 23.2 | MFU 7.44% | FLOPs estimator: workload-aware | logical tokens/s: 74409.5 | valid tokens/s: 74409.5 | physical tokens/s: 75351.7 | loss tokens/s: 54404.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.463802E-01 | lm-global-supervised-tokens: 1.226659E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.393156E-02 | lm-token-buffer-mib: 2.367502E+02 | global_load_balancing_loss: 1.000413E+00 | loss scale: 1.0 | grad norm: 0.140 | params norm: 11155.974 | params rms: 0.060406 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:37:42.903018 | finish at 2026-10-17 19:35:36 + [2026-10-16 12:01:58] iteration 1880/ 2384 | consumed samples: 60160 | elapsed time per iteration (ms): 244677.5 | throughput per GPU (TFLOP/s/GPU): 18.3 | MFU 5.87% | FLOPs estimator: workload-aware | logical tokens/s: 68568.7 | valid tokens/s: 68568.7 | physical tokens/s: 69449.1 | loss tokens/s: 52418.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.944299E-01 | lm-global-supervised-tokens: 1.282560E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.818819E-02 | lm-token-buffer-mib: 2.362838E+02 | global_load_balancing_loss: 1.001350E+00 | loss scale: 1.0 | grad norm: 0.244 | params norm: 11155.943 | params rms: 0.060406 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:15:17.471781 | finish at 2026-10-17 22:17:16 +times across ranks (ms): + save-checkpoint: + rank 0: 25610.91 + rank 1: 25610.89 + rank 2: 25610.90 + rank 3: 25610.90 + rank 4: 25610.88 + rank 5: 25610.96 + rank 6: 25610.90 + rank 7: 25610.73 + rank 8: 25611.30 + rank 9: 25611.32 + rank 10: 25611.33 + rank 11: 25611.29 + rank 12: 25611.29 + rank 13: 25611.30 + rank 14: 25611.29 + rank 15: 25611.30 + rank 16: 25611.10 + rank 17: 25611.12 + rank 18: 25611.08 + rank 19: 25611.13 + rank 20: 25611.07 + rank 21: 25611.07 + rank 22: 25611.14 + rank 23: 25611.06 + rank 24: 25611.48 + rank 25: 25611.47 + rank 26: 25611.54 + rank 27: 25611.61 + rank 28: 25611.47 + rank 29: 25611.49 + rank 30: 25611.48 + rank 31: 25611.52 + rank 32: 25611.17 + rank 33: 25611.16 + rank 34: 25611.28 + rank 35: 25611.14 + rank 36: 25611.12 + rank 37: 25611.27 + rank 38: 25611.12 + rank 39: 25611.12 + rank 40: 25611.50 + rank 41: 25611.49 + rank 42: 25611.50 + rank 43: 25611.50 + rank 44: 25611.50 + rank 45: 25611.51 + rank 46: 25611.46 + rank 47: 25611.48 + rank 48: 25611.57 + rank 49: 25611.60 + rank 50: 25611.67 + rank 51: 25611.56 + rank 52: 25611.71 + rank 53: 25611.67 + rank 54: 25611.65 + rank 55: 25611.55 + rank 56: 25611.29 + rank 57: 25611.34 + rank 58: 25611.33 + rank 59: 25611.34 + rank 60: 25611.33 + rank 61: 25611.29 + rank 62: 25611.32 + rank 63: 25611.28 + [2026-10-16 12:06:22] iteration 1881/ 2384 | consumed samples: 60192 | elapsed time per iteration (ms): 237773.4 | throughput per GPU (TFLOP/s/GPU): 19.3 | MFU 6.20% | FLOPs estimator: workload-aware | logical tokens/s: 70559.6 | valid tokens/s: 70559.5 | physical tokens/s: 71085.6 | loss tokens/s: 53268.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.552338E-01 | lm-global-supervised-tokens: 1.266591E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.432596E-02 | lm-token-buffer-mib: 2.343615E+02 | global_load_balancing_loss: 1.001514E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11155.913 | params rms: 0.060406 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:13:20.039662 | finish at 2026-10-17 21:19:42 + [2026-10-16 12:10:10] iteration 1882/ 2384 | consumed samples: 60224 | elapsed time per iteration (ms): 228359.1 | throughput per GPU (TFLOP/s/GPU): 24.4 | MFU 7.83% | FLOPs estimator: workload-aware | logical tokens/s: 73468.3 | valid tokens/s: 73468.3 | physical tokens/s: 74156.1 | loss tokens/s: 52111.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.651234E-01 | lm-global-supervised-tokens: 1.190020E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.540768E-02 | lm-token-buffer-mib: 2.339299E+02 | global_load_balancing_loss: 1.001334E+00 | loss scale: 1.0 | grad norm: 0.248 | params norm: 11155.882 | params rms: 0.060405 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:50:36.246589 | finish at 2026-10-17 20:00:47 + [2026-10-16 12:13:51] iteration 1883/ 2384 | consumed samples: 60256 | elapsed time per iteration (ms): 220244.0 | throughput per GPU (TFLOP/s/GPU): 20.2 | MFU 6.46% | FLOPs estimator: workload-aware | logical tokens/s: 76175.6 | valid tokens/s: 76175.6 | physical tokens/s: 76984.0 | loss tokens/s: 58779.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.566153E-01 | lm-global-supervised-tokens: 1.294576E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.818662E-02 | lm-token-buffer-mib: 2.338928E+02 | global_load_balancing_loss: 1.000308E+00 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11155.850 | params rms: 0.060405 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:39:02.224867 | finish at 2026-10-17 18:52:53 + [2026-10-16 12:17:31] iteration 1884/ 2384 | consumed samples: 60288 | elapsed time per iteration (ms): 220881.1 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.06% | FLOPs estimator: workload-aware | logical tokens/s: 75955.7 | valid tokens/s: 75955.7 | physical tokens/s: 76805.7 | loss tokens/s: 56531.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.516803E-01 | lm-global-supervised-tokens: 1.248672E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.503604E-02 | lm-token-buffer-mib: 2.345376E+02 | global_load_balancing_loss: 1.000705E+00 | loss scale: 1.0 | grad norm: 0.161 | params norm: 11155.819 | params rms: 0.060405 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:40:40.528512 | finish at 2026-10-17 18:58:12 + [2026-10-16 12:21:12] iteration 1885/ 2384 | consumed samples: 60320 | elapsed time per iteration (ms): 220650.3 | throughput per GPU (TFLOP/s/GPU): 18.6 | MFU 5.95% | FLOPs estimator: workload-aware | logical tokens/s: 76035.3 | valid tokens/s: 76035.3 | physical tokens/s: 76744.8 | loss tokens/s: 58961.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.514304E-01 | lm-global-supervised-tokens: 1.300996E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.637024E-02 | lm-token-buffer-mib: 2.348708E+02 | global_load_balancing_loss: 9.996489E-01 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11155.789 | params rms: 0.060405 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:35:04.500903 | finish at 2026-10-17 18:56:17 + [2026-10-16 12:24:57] iteration 1886/ 2384 | consumed samples: 60352 | elapsed time per iteration (ms): 224999.6 | throughput per GPU (TFLOP/s/GPU): 19.7 | MFU 6.30% | FLOPs estimator: workload-aware | logical tokens/s: 74565.5 | valid tokens/s: 74565.5 | physical tokens/s: 75762.8 | loss tokens/s: 58165.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.902153E-01 | lm-global-supervised-tokens: 1.308723E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.293940E-02 | lm-token-buffer-mib: 2.373205E+02 | global_load_balancing_loss: 1.000111E+00 | loss scale: 1.0 | grad norm: 0.164 | params norm: 11155.760 | params rms: 0.060405 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:07:29.794830 | finish at 2026-10-17 19:32:27 + [2026-10-16 12:28:43] iteration 1887/ 2384 | consumed samples: 60384 | elapsed time per iteration (ms): 225539.2 | throughput per GPU (TFLOP/s/GPU): 27.1 | MFU 8.69% | FLOPs estimator: workload-aware | logical tokens/s: 74387.0 | valid tokens/s: 74387.0 | physical tokens/s: 74833.2 | loss tokens/s: 54839.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.520191E-01 | lm-global-supervised-tokens: 1.236849E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.752296E-02 | lm-token-buffer-mib: 2.339738E+02 | global_load_balancing_loss: 1.000554E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11155.733 | params rms: 0.060405 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:08:13.006606 | finish at 2026-10-17 19:36:56 + [2026-10-16 12:32:21] iteration 1888/ 2384 | consumed samples: 60416 | elapsed time per iteration (ms): 218532.1 | throughput per GPU (TFLOP/s/GPU): 19.7 | MFU 6.31% | FLOPs estimator: workload-aware | logical tokens/s: 76772.2 | valid tokens/s: 76772.2 | physical tokens/s: 77370.0 | loss tokens/s: 58957.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.430929E-01 | lm-global-supervised-tokens: 1.288402E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.444339E-02 | lm-token-buffer-mib: 2.328550E+02 | global_load_balancing_loss: 9.990982E-01 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11155.704 | params rms: 0.060405 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:06:31.929504 | finish at 2026-10-17 18:38:53 + [2026-10-16 12:36:03] iteration 1889/ 2384 | consumed samples: 60448 | elapsed time per iteration (ms): 221585.6 | throughput per GPU (TFLOP/s/GPU): 25.6 | MFU 8.21% | FLOPs estimator: workload-aware | logical tokens/s: 75714.2 | valid tokens/s: 75714.2 | physical tokens/s: 76231.7 | loss tokens/s: 60442.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.965471E-01 | lm-global-supervised-tokens: 1.339328E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.321926E-02 | lm-token-buffer-mib: 2.323961E+02 | global_load_balancing_loss: 9.990339E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11155.674 | params rms: 0.060404 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:28:04.867584 | finish at 2026-10-17 19:04:08 + [2026-10-16 12:40:06] iteration 1890/ 2384 | consumed samples: 60480 | elapsed time per iteration (ms): 243515.6 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.34% | FLOPs estimator: workload-aware | logical tokens/s: 68895.8 | valid tokens/s: 68895.8 | physical tokens/s: 69931.1 | loss tokens/s: 51184.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.043231E-01 | lm-global-supervised-tokens: 1.246417E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.218910E-02 | lm-token-buffer-mib: 2.332724E+02 | global_load_balancing_loss: 1.000039E+00 | loss scale: 1.0 | grad norm: 0.096 | params norm: 11155.642 | params rms: 0.060404 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:24:56.692721 | finish at 2026-10-17 22:05:03 + [2026-10-16 12:43:51] iteration 1891/ 2384 | consumed samples: 60512 | elapsed time per iteration (ms): 224377.8 | throughput per GPU (TFLOP/s/GPU): 25.2 | MFU 8.06% | FLOPs estimator: workload-aware | logical tokens/s: 74772.0 | valid tokens/s: 74772.0 | physical tokens/s: 75130.7 | loss tokens/s: 58518.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.143646E-01 | lm-global-supervised-tokens: 1.313017E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.270962E-02 | lm-token-buffer-mib: 2.325093E+02 | global_load_balancing_loss: 1.000407E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11155.609 | params rms: 0.060404 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:43:38.246108 | finish at 2026-10-17 19:27:29 + [2026-10-16 12:47:32] iteration 1892/ 2384 | consumed samples: 60544 | elapsed time per iteration (ms): 221639.2 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.41% | FLOPs estimator: workload-aware | logical tokens/s: 75695.8 | valid tokens/s: 75695.8 | physical tokens/s: 76207.8 | loss tokens/s: 58692.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.348968E-01 | lm-global-supervised-tokens: 1.300855E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.519378E-02 | lm-token-buffer-mib: 2.342342E+02 | global_load_balancing_loss: 1.000890E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11155.575 | params rms: 0.060404 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:17:26.479231 | finish at 2026-10-17 19:04:59 + [2026-10-16 12:51:14] iteration 1893/ 2384 | consumed samples: 60576 | elapsed time per iteration (ms): 221385.9 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.57% | FLOPs estimator: workload-aware | logical tokens/s: 75782.4 | valid tokens/s: 75782.4 | physical tokens/s: 76044.6 | loss tokens/s: 50735.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.843380E-01 | lm-global-supervised-tokens: 1.123218E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.603576E-02 | lm-token-buffer-mib: 2.328928E+02 | global_load_balancing_loss: 1.001428E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11155.544 | params rms: 0.060404 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:11:40.482646 | finish at 2026-10-17 19:02:54 + [2026-10-16 12:54:58] iteration 1894/ 2384 | consumed samples: 60608 | elapsed time per iteration (ms): 224107.3 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.13% | FLOPs estimator: workload-aware | logical tokens/s: 74862.2 | valid tokens/s: 74862.2 | physical tokens/s: 75574.9 | loss tokens/s: 56512.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.585781E-01 | lm-global-supervised-tokens: 1.266494E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.491943E-02 | lm-token-buffer-mib: 2.342471E+02 | global_load_balancing_loss: 1.002302E+00 | loss scale: 1.0 | grad norm: 0.161 | params norm: 11155.514 | params rms: 0.060403 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:30:12.592442 | finish at 2026-10-17 19:25:10 + [2026-10-16 12:58:39] iteration 1895/ 2384 | consumed samples: 60640 | elapsed time per iteration (ms): 221506.9 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.80% | FLOPs estimator: workload-aware | logical tokens/s: 75741.1 | valid tokens/s: 75741.1 | physical tokens/s: 76483.0 | loss tokens/s: 56550.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.220423E-01 | lm-global-supervised-tokens: 1.252637E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.590452E-02 | lm-token-buffer-mib: 2.336807E+02 | global_load_balancing_loss: 1.000829E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11155.485 | params rms: 0.060403 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:05:16.891740 | finish at 2026-10-17 19:03:56 + [2026-10-16 13:02:27] iteration 1896/ 2384 | consumed samples: 60672 | elapsed time per iteration (ms): 227274.3 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.69% | FLOPs estimator: workload-aware | logical tokens/s: 73322.1 | valid tokens/s: 73322.1 | physical tokens/s: 74532.8 | loss tokens/s: 56543.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.777818E-01 | lm-global-supervised-tokens: 1.285080E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.978364E-02 | lm-token-buffer-mib: 2.337254E+02 | global_load_balancing_loss: 1.001147E+00 | loss scale: 1.0 | grad norm: 0.113 | params norm: 11155.454 | params rms: 0.060403 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:48:29.864964 | finish at 2026-10-17 19:50:56 + [2026-10-16 13:06:09] iteration 1897/ 2384 | consumed samples: 60704 | elapsed time per iteration (ms): 222047.7 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.87% | FLOPs estimator: workload-aware | logical tokens/s: 75556.6 | valid tokens/s: 75556.6 | physical tokens/s: 76352.0 | loss tokens/s: 57422.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.623351E-01 | lm-global-supervised-tokens: 1.275061E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.428804E-02 | lm-token-buffer-mib: 2.325727E+02 | global_load_balancing_loss: 9.992895E-01 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11155.422 | params rms: 0.060403 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:02:17.231026 | finish at 2026-10-17 19:08:26 + [2026-10-16 13:09:49] iteration 1898/ 2384 | consumed samples: 60736 | elapsed time per iteration (ms): 220432.7 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.34% | FLOPs estimator: workload-aware | logical tokens/s: 76110.3 | valid tokens/s: 76110.2 | physical tokens/s: 76561.0 | loss tokens/s: 60273.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.181156E-01 | lm-global-supervised-tokens: 1.328620E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.420554E-02 | lm-token-buffer-mib: 2.340612E+02 | global_load_balancing_loss: 9.977868E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11155.392 | params rms: 0.060403 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:45:30.284745 | finish at 2026-10-17 18:55:19 + [2026-10-16 13:13:29] iteration 1899/ 2384 | consumed samples: 60768 | elapsed time per iteration (ms): 219506.3 | throughput per GPU (TFLOP/s/GPU): 23.7 | MFU 7.58% | FLOPs estimator: workload-aware | logical tokens/s: 76431.4 | valid tokens/s: 76431.4 | physical tokens/s: 77035.4 | loss tokens/s: 58544.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.289098E-01 | lm-global-supervised-tokens: 1.285095E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.999584E-02 | lm-token-buffer-mib: 2.327939E+02 | global_load_balancing_loss: 9.988712E-01 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11155.363 | params rms: 0.060403 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:34:20.574566 | finish at 2026-10-17 18:47:49 + [2026-10-16 13:17:32] iteration 1900/ 2384 | consumed samples: 60800 | elapsed time per iteration (ms): 243760.3 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.39% | FLOPs estimator: workload-aware | logical tokens/s: 68826.6 | valid tokens/s: 68826.6 | physical tokens/s: 69327.9 | loss tokens/s: 51092.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.260226E-01 | lm-global-supervised-tokens: 1.245425E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.284792E-02 | lm-token-buffer-mib: 2.337204E+02 | global_load_balancing_loss: 9.990971E-01 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11155.334 | params rms: 0.060403 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:46:19.967045 | finish at 2026-10-17 22:03:52 + [2026-10-16 13:21:17] iteration 1901/ 2384 | consumed samples: 60832 | elapsed time per iteration (ms): 224559.8 | throughput per GPU (TFLOP/s/GPU): 24.7 | MFU 7.91% | FLOPs estimator: workload-aware | logical tokens/s: 74711.4 | valid tokens/s: 74711.4 | physical tokens/s: 75235.1 | loss tokens/s: 53941.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.598559E-01 | lm-global-supervised-tokens: 1.211313E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.590185E-02 | lm-token-buffer-mib: 2.335574E+02 | global_load_balancing_loss: 1.000190E+00 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11155.304 | params rms: 0.060402 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:07:42.369538 | finish at 2026-10-17 19:28:59 + [2026-10-16 13:24:56] iteration 1902/ 2384 | consumed samples: 60864 | elapsed time per iteration (ms): 219338.3 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.02% | FLOPs estimator: workload-aware | logical tokens/s: 76490.0 | valid tokens/s: 76490.0 | physical tokens/s: 77680.0 | loss tokens/s: 54959.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.333076E-01 | lm-global-supervised-tokens: 1.205466E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.767521E-02 | lm-token-buffer-mib: 2.334333E+02 | global_load_balancing_loss: 1.001457E+00 | loss scale: 1.0 | grad norm: 0.218 | params norm: 11155.272 | params rms: 0.060402 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:22:01.083004 | finish at 2026-10-17 18:46:57 + [2026-10-16 13:28:38] iteration 1903/ 2384 | consumed samples: 60896 | elapsed time per iteration (ms): 221797.1 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.33% | FLOPs estimator: workload-aware | logical tokens/s: 75642.1 | valid tokens/s: 75642.1 | physical tokens/s: 76132.4 | loss tokens/s: 55802.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.361286E-01 | lm-global-supervised-tokens: 1.237689E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.564441E-02 | lm-token-buffer-mib: 2.327645E+02 | global_load_balancing_loss: 1.002441E+00 | loss scale: 1.0 | grad norm: 0.162 | params norm: 11155.241 | params rms: 0.060402 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:38:04.426004 | finish at 2026-10-17 19:06:42 + [2026-10-16 13:32:24] iteration 1904/ 2384 | consumed samples: 60928 | elapsed time per iteration (ms): 226142.2 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.99% | FLOPs estimator: workload-aware | logical tokens/s: 74053.2 | valid tokens/s: 74053.2 | physical tokens/s: 74775.2 | loss tokens/s: 57406.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.014263E-01 | lm-global-supervised-tokens: 1.298211E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.289868E-02 | lm-token-buffer-mib: 2.345094E+02 | global_load_balancing_loss: 1.001321E+00 | loss scale: 1.0 | grad norm: 0.387 | params norm: 11155.212 | params rms: 0.060402 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:09:08.232651 | finish at 2026-10-17 19:41:32 + [2026-10-16 13:36:07] iteration 1905/ 2384 | consumed samples: 60960 | elapsed time per iteration (ms): 223083.5 | throughput per GPU (TFLOP/s/GPU): 23.4 | MFU 7.50% | FLOPs estimator: workload-aware | logical tokens/s: 75205.9 | valid tokens/s: 75205.9 | physical tokens/s: 75967.7 | loss tokens/s: 50121.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.662739E-01 | lm-global-supervised-tokens: 1.118136E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.627808E-02 | lm-token-buffer-mib: 2.333212E+02 | global_load_balancing_loss: 1.001493E+00 | loss scale: 1.0 | grad norm: 0.151 | params norm: 11155.181 | params rms: 0.060402 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:40:57.020324 | finish at 2026-10-17 19:17:04 + [2026-10-16 13:39:49] iteration 1906/ 2384 | consumed samples: 60992 | elapsed time per iteration (ms): 221490.7 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.05% | FLOPs estimator: workload-aware | logical tokens/s: 75746.8 | valid tokens/s: 75746.8 | physical tokens/s: 76329.3 | loss tokens/s: 52040.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.070637E-01 | lm-global-supervised-tokens: 1.152647E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.916865E-02 | lm-token-buffer-mib: 2.327356E+02 | global_load_balancing_loss: 1.000005E+00 | loss scale: 1.0 | grad norm: 0.238 | params norm: 11155.151 | params rms: 0.060402 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:24:32.576484 | finish at 2026-10-17 19:04:21 + [2026-10-16 13:43:33] iteration 1907/ 2384 | consumed samples: 61024 | elapsed time per iteration (ms): 224702.9 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.10% | FLOPs estimator: workload-aware | logical tokens/s: 74664.0 | valid tokens/s: 74664.0 | physical tokens/s: 75266.1 | loss tokens/s: 54794.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.797282E-01 | lm-global-supervised-tokens: 1.231236E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.209049E-02 | lm-token-buffer-mib: 2.350680E+02 | global_load_balancing_loss: 9.991000E-01 | loss scale: 1.0 | grad norm: 0.333 | params norm: 11155.122 | params rms: 0.060401 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:46:23.275307 | finish at 2026-10-17 19:29:57 + [2026-10-16 13:47:17] iteration 1908/ 2384 | consumed samples: 61056 | elapsed time per iteration (ms): 223304.5 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.80% | FLOPs estimator: workload-aware | logical tokens/s: 75131.5 | valid tokens/s: 75131.5 | physical tokens/s: 76107.7 | loss tokens/s: 54855.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.824656E-01 | lm-global-supervised-tokens: 1.224952E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.675312E-02 | lm-token-buffer-mib: 2.347160E+02 | global_load_balancing_loss: 1.000069E+00 | loss scale: 1.0 | grad norm: 0.252 | params norm: 11155.091 | params rms: 0.060401 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:31:32.937963 | finish at 2026-10-17 19:18:50 + [2026-10-16 13:51:05] iteration 1909/ 2384 | consumed samples: 61088 | elapsed time per iteration (ms): 227818.6 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.11% | FLOPs estimator: workload-aware | logical tokens/s: 73642.7 | valid tokens/s: 73642.7 | physical tokens/s: 75058.7 | loss tokens/s: 59063.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.265426E-01 | lm-global-supervised-tokens: 1.345569E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.414818E-02 | lm-token-buffer-mib: 2.367993E+02 | global_load_balancing_loss: 1.002383E+00 | loss scale: 1.0 | grad norm: 0.388 | params norm: 11155.058 | params rms: 0.060401 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:03:33.813907 | finish at 2026-10-17 19:54:38 + [2026-10-16 13:55:09] iteration 1910/ 2384 | consumed samples: 61120 | elapsed time per iteration (ms): 244109.4 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.42% | FLOPs estimator: workload-aware | logical tokens/s: 68727.9 | valid tokens/s: 68727.9 | physical tokens/s: 69145.9 | loss tokens/s: 51476.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.680138E-01 | lm-global-supervised-tokens: 1.256600E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.666241E-02 | lm-token-buffer-mib: 2.333857E+02 | global_load_balancing_loss: 1.000584E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11155.030 | params rms: 0.060401 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:08:27.867708 | finish at 2026-10-17 22:03:37 + [2026-10-16 13:58:56] iteration 1911/ 2384 | consumed samples: 61152 | elapsed time per iteration (ms): 227315.8 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.37% | FLOPs estimator: workload-aware | logical tokens/s: 73805.6 | valid tokens/s: 73805.6 | physical tokens/s: 74316.5 | loss tokens/s: 54259.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.070041E-01 | lm-global-supervised-tokens: 1.233401E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.436369E-02 | lm-token-buffer-mib: 2.345091E+02 | global_load_balancing_loss: 1.000645E+00 | loss scale: 1.0 | grad norm: 0.105 | params norm: 11155.004 | params rms: 0.060401 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:52:00.384993 | finish at 2026-10-17 19:50:56 + [2026-10-16 14:02:36] iteration 1912/ 2384 | consumed samples: 61184 | elapsed time per iteration (ms): 219674.6 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.73% | FLOPs estimator: workload-aware | logical tokens/s: 76372.9 | valid tokens/s: 76372.9 | physical tokens/s: 76754.1 | loss tokens/s: 56869.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.726805E-01 | lm-global-supervised-tokens: 1.249276E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.155299E-02 | lm-token-buffer-mib: 2.322708E+02 | global_load_balancing_loss: 1.000170E+00 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11154.974 | params rms: 0.060401 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:48:06.415873 | finish at 2026-10-17 18:50:42 + [2026-10-16 14:06:17] iteration 1913/ 2384 | consumed samples: 61216 | elapsed time per iteration (ms): 221866.8 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.75% | FLOPs estimator: workload-aware | logical tokens/s: 75618.2 | valid tokens/s: 75618.2 | physical tokens/s: 76126.4 | loss tokens/s: 53002.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.070940E-01 | lm-global-supervised-tokens: 1.175937E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.247718E-02 | lm-token-buffer-mib: 2.345566E+02 | global_load_balancing_loss: 9.988826E-01 | loss scale: 1.0 | grad norm: 0.143 | params norm: 11154.940 | params rms: 0.060400 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:01:39.249807 | finish at 2026-10-17 19:07:57 + [2026-10-16 14:09:56] iteration 1914/ 2384 | consumed samples: 61248 | elapsed time per iteration (ms): 218296.6 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.97% | FLOPs estimator: workload-aware | logical tokens/s: 76855.1 | valid tokens/s: 76855.1 | physical tokens/s: 77172.9 | loss tokens/s: 55514.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.566443E-01 | lm-global-supervised-tokens: 1.211857E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.438929E-02 | lm-token-buffer-mib: 2.329854E+02 | global_load_balancing_loss: 9.995880E-01 | loss scale: 1.0 | grad norm: 0.110 | params norm: 11154.905 | params rms: 0.060400 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:29:59.406531 | finish at 2026-10-17 18:39:55 + [2026-10-16 14:13:40] iteration 1915/ 2384 | consumed samples: 61280 | elapsed time per iteration (ms): 223859.4 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.32% | FLOPs estimator: workload-aware | logical tokens/s: 74945.2 | valid tokens/s: 74945.2 | physical tokens/s: 75473.7 | loss tokens/s: 54618.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.108145E-01 | lm-global-supervised-tokens: 1.222686E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.377782E-02 | lm-token-buffer-mib: 2.337812E+02 | global_load_balancing_loss: 1.001135E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11154.873 | params rms: 0.060400 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:09:50.057833 | finish at 2026-10-17 19:23:30 + [2026-10-16 14:17:20] iteration 1916/ 2384 | consumed samples: 61312 | elapsed time per iteration (ms): 220300.2 | throughput per GPU (TFLOP/s/GPU): 23.2 | MFU 7.44% | FLOPs estimator: workload-aware | logical tokens/s: 76156.1 | valid tokens/s: 76156.1 | physical tokens/s: 76895.8 | loss tokens/s: 57429.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.773759E-01 | lm-global-supervised-tokens: 1.265177E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.831144E-02 | lm-token-buffer-mib: 2.339507E+02 | global_load_balancing_loss: 1.001457E+00 | loss scale: 1.0 | grad norm: 0.121 | params norm: 11154.843 | params rms: 0.060400 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:38:20.514459 | finish at 2026-10-17 18:55:40 + [2026-10-16 14:21:02] iteration 1917/ 2384 | consumed samples: 61344 | elapsed time per iteration (ms): 222272.7 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.57% | FLOPs estimator: workload-aware | logical tokens/s: 75480.3 | valid tokens/s: 75480.3 | physical tokens/s: 75855.2 | loss tokens/s: 52074.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.579583E-01 | lm-global-supervised-tokens: 1.157476E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.188859E-02 | lm-token-buffer-mib: 2.326290E+02 | global_load_balancing_loss: 1.002099E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11154.810 | params rms: 0.060400 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:50:01.331115 | finish at 2026-10-17 19:11:04 + [2026-10-16 14:24:46] iteration 1918/ 2384 | consumed samples: 61376 | elapsed time per iteration (ms): 223715.1 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.40% | FLOPs estimator: workload-aware | logical tokens/s: 74993.5 | valid tokens/s: 74993.5 | physical tokens/s: 75559.5 | loss tokens/s: 56091.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.700852E-01 | lm-global-supervised-tokens: 1.254851E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.147783E-02 | lm-token-buffer-mib: 2.345804E+02 | global_load_balancing_loss: 1.000975E+00 | loss scale: 1.0 | grad norm: 0.196 | params norm: 11154.780 | params rms: 0.060400 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:57:31.249178 | finish at 2026-10-17 19:22:17 + [2026-10-16 14:28:28] iteration 1919/ 2384 | consumed samples: 61408 | elapsed time per iteration (ms): 221591.4 | throughput per GPU (TFLOP/s/GPU): 25.3 | MFU 8.09% | FLOPs estimator: workload-aware | logical tokens/s: 75712.3 | valid tokens/s: 75712.3 | physical tokens/s: 76157.2 | loss tokens/s: 51911.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.475169E-01 | lm-global-supervised-tokens: 1.150317E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.414265E-02 | lm-token-buffer-mib: 2.336403E+02 | global_load_balancing_loss: 1.000774E+00 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11154.751 | params rms: 0.060399 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:37:20.023795 | finish at 2026-10-17 19:05:48 + [2026-10-16 14:32:35] iteration 1920/ 2384 | consumed samples: 61440 | elapsed time per iteration (ms): 247375.9 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.60% | FLOPs estimator: workload-aware | logical tokens/s: 67820.7 | valid tokens/s: 67820.7 | physical tokens/s: 68561.4 | loss tokens/s: 52671.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.966212E-01 | lm-global-supervised-tokens: 1.302972E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.356033E-02 | lm-token-buffer-mib: 2.356852E+02 | global_load_balancing_loss: 9.995731E-01 | loss scale: 1.0 | grad norm: 0.154 | params norm: 11154.721 | params rms: 0.060399 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:53:02.414627 | finish at 2026-10-17 22:25:37 +times across ranks (ms): + save-checkpoint: + rank 0: 24036.11 + rank 1: 24036.12 + rank 2: 24036.13 + rank 3: 24036.11 + rank 4: 24036.12 + rank 5: 24036.13 + rank 6: 24036.13 + rank 7: 24036.14 + rank 8: 24035.95 + rank 9: 24035.99 + rank 10: 24035.95 + rank 11: 24035.95 + rank 12: 24036.03 + rank 13: 24035.93 + rank 14: 24035.95 + rank 15: 24035.95 + rank 16: 24035.98 + rank 17: 24035.97 + rank 18: 24035.97 + rank 19: 24036.02 + rank 20: 24035.95 + rank 21: 24035.95 + rank 22: 24036.00 + rank 23: 24035.96 + rank 24: 24035.76 + rank 25: 24035.57 + rank 26: 24035.81 + rank 27: 24035.60 + rank 28: 24035.56 + rank 29: 24035.59 + rank 30: 24035.57 + rank 31: 24035.58 + rank 32: 24037.08 + rank 33: 24037.08 + rank 34: 24037.04 + rank 35: 24037.07 + rank 36: 24037.03 + rank 37: 24037.09 + rank 38: 24037.02 + rank 39: 24037.05 + rank 40: 24037.41 + rank 41: 24037.32 + rank 42: 24037.32 + rank 43: 24037.35 + rank 44: 24037.29 + rank 45: 24037.32 + rank 46: 24037.31 + rank 47: 24037.29 + rank 48: 24036.49 + rank 49: 24036.52 + rank 50: 24036.54 + rank 51: 24036.50 + rank 52: 24036.47 + rank 53: 24036.51 + rank 54: 24036.47 + rank 55: 24036.49 + rank 56: 24036.61 + rank 57: 24036.59 + rank 58: 24036.60 + rank 59: 24036.59 + rank 60: 24036.59 + rank 61: 24036.58 + rank 62: 24036.56 + rank 63: 24036.57 + [2026-10-16 14:36:41] iteration 1921/ 2384 | consumed samples: 61472 | elapsed time per iteration (ms): 222070.4 | throughput per GPU (TFLOP/s/GPU): 24.4 | MFU 7.81% | FLOPs estimator: workload-aware | logical tokens/s: 75548.9 | valid tokens/s: 75548.9 | physical tokens/s: 76141.6 | loss tokens/s: 60596.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.115759E-01 | lm-global-supervised-tokens: 1.345674E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.298804E-02 | lm-token-buffer-mib: 2.326775E+02 | global_load_balancing_loss: 9.994186E-01 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11154.687 | params rms: 0.060399 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:33:38.603369 | finish at 2026-10-17 19:10:20 + [2026-10-16 14:40:28] iteration 1922/ 2384 | consumed samples: 61504 | elapsed time per iteration (ms): 227279.2 | throughput per GPU (TFLOP/s/GPU): 25.0 | MFU 8.02% | FLOPs estimator: workload-aware | logical tokens/s: 73817.7 | valid tokens/s: 73817.7 | physical tokens/s: 74200.6 | loss tokens/s: 57645.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.375400E-01 | lm-global-supervised-tokens: 1.310160E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.545688E-02 | lm-token-buffer-mib: 2.335052E+02 | global_load_balancing_loss: 1.002036E+00 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11154.652 | params rms: 0.060399 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:10:02.986030 | finish at 2026-10-17 19:50:31 + [2026-10-16 14:44:09] iteration 1923/ 2384 | consumed samples: 61536 | elapsed time per iteration (ms): 220671.2 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.55% | FLOPs estimator: workload-aware | logical tokens/s: 76028.0 | valid tokens/s: 76028.0 | physical tokens/s: 76890.4 | loss tokens/s: 62178.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.901527E-01 | lm-global-supervised-tokens: 1.372102E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.548064E-02 | lm-token-buffer-mib: 2.327388E+02 | global_load_balancing_loss: 9.997468E-01 | loss scale: 1.0 | grad norm: 0.107 | params norm: 11154.619 | params rms: 0.060399 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:15:29.405192 | finish at 2026-10-17 18:59:38 + [2026-10-16 14:47:51] iteration 1924/ 2384 | consumed samples: 61568 | elapsed time per iteration (ms): 221608.8 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.40% | FLOPs estimator: workload-aware | logical tokens/s: 75706.2 | valid tokens/s: 75706.2 | physical tokens/s: 77001.7 | loss tokens/s: 61481.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.918244E-01 | lm-global-supervised-tokens: 1.362493E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.564422E-02 | lm-token-buffer-mib: 2.362472E+02 | global_load_balancing_loss: 9.992981E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11154.589 | params rms: 0.060398 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:19:00.033932 | finish at 2026-10-17 19:06:51 + [2026-10-16 14:51:31] iteration 1925/ 2384 | consumed samples: 61600 | elapsed time per iteration (ms): 219916.4 | throughput per GPU (TFLOP/s/GPU): 19.6 | MFU 6.28% | FLOPs estimator: workload-aware | logical tokens/s: 76289.0 | valid tokens/s: 76289.0 | physical tokens/s: 77155.1 | loss tokens/s: 59532.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.446249E-01 | lm-global-supervised-tokens: 1.309216E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.192056E-02 | lm-token-buffer-mib: 2.337920E+02 | global_load_balancing_loss: 9.995452E-01 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11154.558 | params rms: 0.060398 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:02:21.635021 | finish at 2026-10-17 18:53:52 + [2026-10-16 14:55:16] iteration 1926/ 2384 | consumed samples: 61632 | elapsed time per iteration (ms): 225085.6 | throughput per GPU (TFLOP/s/GPU): 18.6 | MFU 5.96% | FLOPs estimator: workload-aware | logical tokens/s: 74536.9 | valid tokens/s: 74536.9 | physical tokens/s: 75542.7 | loss tokens/s: 58141.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.165924E-01 | lm-global-supervised-tokens: 1.308690E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.779066E-02 | lm-token-buffer-mib: 2.345936E+02 | global_load_balancing_loss: 9.999164E-01 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11154.522 | params rms: 0.060398 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:38:09.198967 | finish at 2026-10-17 19:33:25 + [2026-10-16 14:58:56] iteration 1927/ 2384 | consumed samples: 61664 | elapsed time per iteration (ms): 220565.1 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.20% | FLOPs estimator: workload-aware | logical tokens/s: 76064.5 | valid tokens/s: 76064.5 | physical tokens/s: 76744.5 | loss tokens/s: 59682.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.765848E-01 | lm-global-supervised-tokens: 1.316394E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.959996E-02 | lm-token-buffer-mib: 2.341419E+02 | global_load_balancing_loss: 9.995072E-01 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11154.486 | params rms: 0.060398 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:59:58.264299 | finish at 2026-10-17 18:58:55 + [2026-10-16 15:02:40] iteration 1928/ 2384 | consumed samples: 61696 | elapsed time per iteration (ms): 223431.4 | throughput per GPU (TFLOP/s/GPU): 23.8 | MFU 7.63% | FLOPs estimator: workload-aware | logical tokens/s: 75088.7 | valid tokens/s: 75088.7 | physical tokens/s: 75929.6 | loss tokens/s: 59259.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.067478E-01 | lm-global-supervised-tokens: 1.324040E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.239530E-02 | lm-token-buffer-mib: 2.334358E+02 | global_load_balancing_loss: 1.000491E+00 | loss scale: 1.0 | grad norm: 0.141 | params norm: 11154.449 | params rms: 0.060398 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:18:04.732344 | finish at 2026-10-17 19:20:44 + [2026-10-16 15:06:24] iteration 1929/ 2384 | consumed samples: 61728 | elapsed time per iteration (ms): 223807.1 | throughput per GPU (TFLOP/s/GPU): 26.3 | MFU 8.43% | FLOPs estimator: workload-aware | logical tokens/s: 74962.8 | valid tokens/s: 74962.7 | physical tokens/s: 75782.7 | loss tokens/s: 54485.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.844590E-01 | lm-global-supervised-tokens: 1.219419E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.571198E-02 | lm-token-buffer-mib: 2.345739E+02 | global_load_balancing_loss: 1.000341E+00 | loss scale: 1.0 | grad norm: 0.149 | params norm: 11154.412 | params rms: 0.060398 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:17:12.215014 | finish at 2026-10-17 19:23:36 + [2026-10-16 15:10:26] iteration 1930/ 2384 | consumed samples: 61760 | elapsed time per iteration (ms): 242699.6 | throughput per GPU (TFLOP/s/GPU): 17.9 | MFU 5.73% | FLOPs estimator: workload-aware | logical tokens/s: 69127.4 | valid tokens/s: 69127.4 | physical tokens/s: 70000.3 | loss tokens/s: 51738.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.332217E-01 | lm-global-supervised-tokens: 1.255692E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.516776E-02 | lm-token-buffer-mib: 2.348447E+02 | global_load_balancing_loss: 1.001099E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11154.374 | params rms: 0.060397 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:36:25.640256 | finish at 2026-10-17 21:46:52 + [2026-10-16 15:14:08] iteration 1931/ 2384 | consumed samples: 61792 | elapsed time per iteration (ms): 221970.1 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.63% | FLOPs estimator: workload-aware | logical tokens/s: 75583.1 | valid tokens/s: 75583.1 | physical tokens/s: 76211.8 | loss tokens/s: 58090.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.393806E-01 | lm-global-supervised-tokens: 1.289429E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.698390E-02 | lm-token-buffer-mib: 2.337276E+02 | global_load_balancing_loss: 1.001859E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11154.337 | params rms: 0.060397 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:55:52.448030 | finish at 2026-10-17 19:10:01 + [2026-10-16 15:17:53] iteration 1932/ 2384 | consumed samples: 61824 | elapsed time per iteration (ms): 225040.6 | throughput per GPU (TFLOP/s/GPU): 18.2 | MFU 5.84% | FLOPs estimator: workload-aware | logical tokens/s: 74551.9 | valid tokens/s: 74551.9 | physical tokens/s: 75303.0 | loss tokens/s: 57527.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.780935E-01 | lm-global-supervised-tokens: 1.294600E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.761037E-02 | lm-token-buffer-mib: 2.337162E+02 | global_load_balancing_loss: 1.001265E+00 | loss scale: 1.0 | grad norm: 0.301 | params norm: 11154.302 | params rms: 0.060397 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:15:18.334632 | finish at 2026-10-17 19:33:12 + [2026-10-16 15:21:39] iteration 1933/ 2384 | consumed samples: 61856 | elapsed time per iteration (ms): 225748.9 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.60% | FLOPs estimator: workload-aware | logical tokens/s: 74317.8 | valid tokens/s: 74317.8 | physical tokens/s: 75534.5 | loss tokens/s: 53574.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.072748E-01 | lm-global-supervised-tokens: 1.209445E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.660406E-02 | lm-token-buffer-mib: 2.350245E+02 | global_load_balancing_loss: 1.000552E+00 | loss scale: 1.0 | grad norm: 0.464 | params norm: 11154.268 | params rms: 0.060397 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:16:52.764409 | finish at 2026-10-17 19:38:32 + [2026-10-16 15:25:22] iteration 1934/ 2384 | consumed samples: 61888 | elapsed time per iteration (ms): 223388.2 | throughput per GPU (TFLOP/s/GPU): 24.1 | MFU 7.73% | FLOPs estimator: workload-aware | logical tokens/s: 75103.2 | valid tokens/s: 75103.2 | physical tokens/s: 75822.5 | loss tokens/s: 59466.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.763932E-01 | lm-global-supervised-tokens: 1.328410E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.430831E-02 | lm-token-buffer-mib: 2.334579E+02 | global_load_balancing_loss: 1.000221E+00 | loss scale: 1.0 | grad norm: 0.127 | params norm: 11154.233 | params rms: 0.060397 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:55:24.705899 | finish at 2026-10-17 19:20:47 + [2026-10-16 15:29:05] iteration 1935/ 2384 | consumed samples: 61920 | elapsed time per iteration (ms): 222378.9 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.04% | FLOPs estimator: workload-aware | logical tokens/s: 75444.0 | valid tokens/s: 75444.0 | physical tokens/s: 75941.6 | loss tokens/s: 52914.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.889365E-01 | lm-global-supervised-tokens: 1.176705E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.012388E-02 | lm-token-buffer-mib: 2.332072E+02 | global_load_balancing_loss: 9.997922E-01 | loss scale: 1.0 | grad norm: 0.114 | params norm: 11154.194 | params rms: 0.060396 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:44:08.118951 | finish at 2026-10-17 19:13:13 + [2026-10-16 15:32:49] iteration 1936/ 2384 | consumed samples: 61952 | elapsed time per iteration (ms): 224212.7 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.84% | FLOPs estimator: workload-aware | logical tokens/s: 74827.1 | valid tokens/s: 74827.1 | physical tokens/s: 75461.2 | loss tokens/s: 55584.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.575691E-01 | lm-global-supervised-tokens: 1.246270E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.773133E-02 | lm-token-buffer-mib: 2.327951E+02 | global_load_balancing_loss: 9.986623E-01 | loss scale: 1.0 | grad norm: 0.151 | params norm: 11154.156 | params rms: 0.060396 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:54:07.286667 | finish at 2026-10-17 19:26:56 + [2026-10-16 15:36:30] iteration 1937/ 2384 | consumed samples: 61984 | elapsed time per iteration (ms): 221320.0 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.68% | FLOPs estimator: workload-aware | logical tokens/s: 75805.2 | valid tokens/s: 75805.1 | physical tokens/s: 76580.8 | loss tokens/s: 55661.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.120797E-01 | lm-global-supervised-tokens: 1.231906E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.357793E-02 | lm-token-buffer-mib: 2.344339E+02 | global_load_balancing_loss: 9.999555E-01 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11154.118 | params rms: 0.060396 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:28:50.060858 | finish at 2026-10-17 19:05:20 + [2026-10-16 15:40:11] iteration 1938/ 2384 | consumed samples: 62016 | elapsed time per iteration (ms): 221114.6 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.33% | FLOPs estimator: workload-aware | logical tokens/s: 75875.5 | valid tokens/s: 75875.5 | physical tokens/s: 76405.6 | loss tokens/s: 59672.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.389888E-01 | lm-global-supervised-tokens: 1.319439E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.332242E-02 | lm-token-buffer-mib: 2.324018E+02 | global_load_balancing_loss: 1.000782E+00 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11154.083 | params rms: 0.060396 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:23:37.113489 | finish at 2026-10-17 19:03:48 + [2026-10-16 15:43:53] iteration 1939/ 2384 | consumed samples: 62048 | elapsed time per iteration (ms): 221728.6 | throughput per GPU (TFLOP/s/GPU): 26.2 | MFU 8.39% | FLOPs estimator: workload-aware | logical tokens/s: 75665.4 | valid tokens/s: 75665.4 | physical tokens/s: 76751.1 | loss tokens/s: 53366.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.798789E-01 | lm-global-supervised-tokens: 1.183278E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.613904E-02 | lm-token-buffer-mib: 2.351981E+02 | global_load_balancing_loss: 1.001660E+00 | loss scale: 1.0 | grad norm: 0.135 | params norm: 11154.050 | params rms: 0.060396 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:24:29.231892 | finish at 2026-10-17 19:08:22 + [2026-10-16 15:47:56] iteration 1940/ 2384 | consumed samples: 62080 | elapsed time per iteration (ms): 242819.5 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.58% | FLOPs estimator: workload-aware | logical tokens/s: 69093.3 | valid tokens/s: 69093.3 | physical tokens/s: 69779.4 | loss tokens/s: 52260.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.289542E-01 | lm-global-supervised-tokens: 1.268984E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.078355E-02 | lm-token-buffer-mib: 2.332166E+02 | global_load_balancing_loss: 1.002274E+00 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11154.016 | params rms: 0.060395 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:56:51.852694 | finish at 2026-10-17 21:44:48 + [2026-10-16 15:51:37] iteration 1941/ 2384 | consumed samples: 62112 | elapsed time per iteration (ms): 221431.2 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.53% | FLOPs estimator: workload-aware | logical tokens/s: 75767.1 | valid tokens/s: 75767.1 | physical tokens/s: 76630.5 | loss tokens/s: 61518.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.102042E-01 | lm-global-supervised-tokens: 1.362208E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.636823E-02 | lm-token-buffer-mib: 2.351321E+02 | global_load_balancing_loss: 1.001275E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11153.982 | params rms: 0.060395 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:14:54.006297 | finish at 2026-10-17 19:06:31 + [2026-10-16 15:55:19] iteration 1942/ 2384 | consumed samples: 62144 | elapsed time per iteration (ms): 221491.6 | throughput per GPU (TFLOP/s/GPU): 25.4 | MFU 8.15% | FLOPs estimator: workload-aware | logical tokens/s: 75746.4 | valid tokens/s: 75746.4 | physical tokens/s: 76119.8 | loss tokens/s: 53506.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.861991E-01 | lm-global-supervised-tokens: 1.185118E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.220028E-02 | lm-token-buffer-mib: 2.328828E+02 | global_load_balancing_loss: 9.994562E-01 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11153.950 | params rms: 0.060395 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:11:39.287111 | finish at 2026-10-17 19:06:58 + [2026-10-16 15:59:01] iteration 1943/ 2384 | consumed samples: 62176 | elapsed time per iteration (ms): 222338.9 | throughput per GPU (TFLOP/s/GPU): 24.1 | MFU 7.73% | FLOPs estimator: workload-aware | logical tokens/s: 75457.6 | valid tokens/s: 75457.6 | physical tokens/s: 76354.5 | loss tokens/s: 58081.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.177632E-01 | lm-global-supervised-tokens: 1.291380E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 5.272308E-02 | lm-token-buffer-mib: 2.339918E+02 | global_load_balancing_loss: 9.996463E-01 | loss scale: 1.0 | grad norm: 0.163 | params norm: 11153.917 | params rms: 0.060395 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:14:11.475758 | finish at 2026-10-17 19:13:13 + [2026-10-16 16:02:43] iteration 1944/ 2384 | consumed samples: 62208 | elapsed time per iteration (ms): 221578.1 | throughput per GPU (TFLOP/s/GPU): 19.7 | MFU 6.31% | FLOPs estimator: workload-aware | logical tokens/s: 75716.7 | valid tokens/s: 75716.7 | physical tokens/s: 76406.7 | loss tokens/s: 59309.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.475143E-01 | lm-global-supervised-tokens: 1.314173E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.763226E-02 | lm-token-buffer-mib: 2.329520E+02 | global_load_balancing_loss: 1.000053E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11153.882 | params rms: 0.060395 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:04:54.350872 | finish at 2026-10-17 19:07:37 + [2026-10-16 16:06:25] iteration 1945/ 2384 | consumed samples: 62240 | elapsed time per iteration (ms): 222358.0 | throughput per GPU (TFLOP/s/GPU): 16.7 | MFU 5.37% | FLOPs estimator: workload-aware | logical tokens/s: 75451.3 | valid tokens/s: 75451.2 | physical tokens/s: 76598.1 | loss tokens/s: 60039.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.073885E-01 | lm-global-supervised-tokens: 1.335033E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.699762E-02 | lm-token-buffer-mib: 2.362674E+02 | global_load_balancing_loss: 9.995694E-01 | loss scale: 1.0 | grad norm: 0.152 | params norm: 11153.851 | params rms: 0.060394 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:06:55.157517 | finish at 2026-10-17 19:13:20 + [2026-10-16 16:10:08] iteration 1946/ 2384 | consumed samples: 62272 | elapsed time per iteration (ms): 222748.2 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.72% | FLOPs estimator: workload-aware | logical tokens/s: 75319.1 | valid tokens/s: 75319.1 | physical tokens/s: 76179.7 | loss tokens/s: 56215.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.842621E-01 | lm-global-supervised-tokens: 1.252193E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.103571E-02 | lm-token-buffer-mib: 2.341769E+02 | global_load_balancing_loss: 1.001239E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11153.819 | params rms: 0.060394 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:06:03.706143 | finish at 2026-10-17 19:16:12 + [2026-10-16 16:13:54] iteration 1947/ 2384 | consumed samples: 62304 | elapsed time per iteration (ms): 225748.8 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.77% | FLOPs estimator: workload-aware | logical tokens/s: 74318.0 | valid tokens/s: 74318.0 | physical tokens/s: 75361.9 | loss tokens/s: 57600.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.896113E-01 | lm-global-supervised-tokens: 1.300320E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.628457E-02 | lm-token-buffer-mib: 2.355405E+02 | global_load_balancing_loss: 1.000996E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11153.785 | params rms: 0.060394 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:24:12.224888 | finish at 2026-10-17 19:38:06 + [2026-10-16 16:17:36] iteration 1948/ 2384 | consumed samples: 62336 | elapsed time per iteration (ms): 221899.5 | throughput per GPU (TFLOP/s/GPU): 20.2 | MFU 6.48% | FLOPs estimator: workload-aware | logical tokens/s: 75607.1 | valid tokens/s: 75607.1 | physical tokens/s: 76042.2 | loss tokens/s: 58578.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.881292E-01 | lm-global-supervised-tokens: 1.299864E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.337527E-02 | lm-token-buffer-mib: 2.341586E+02 | global_load_balancing_loss: 1.002183E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11153.751 | params rms: 0.060394 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:52:28.201808 | finish at 2026-10-17 19:10:04 + [2026-10-16 16:21:22] iteration 1949/ 2384 | consumed samples: 62368 | elapsed time per iteration (ms): 226365.1 | throughput per GPU (TFLOP/s/GPU): 24.7 | MFU 7.91% | FLOPs estimator: workload-aware | logical tokens/s: 74115.6 | valid tokens/s: 74115.6 | physical tokens/s: 74902.0 | loss tokens/s: 57103.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.517172E-01 | lm-global-supervised-tokens: 1.292614E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.906264E-02 | lm-token-buffer-mib: 2.360646E+02 | global_load_balancing_loss: 1.001334E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11153.719 | params rms: 0.060394 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:21:08.819497 | finish at 2026-10-17 19:42:31 + [2026-10-16 16:25:30] iteration 1950/ 2384 | consumed samples: 62400 | elapsed time per iteration (ms): 248507.2 | throughput per GPU (TFLOP/s/GPU): 19.4 | MFU 6.22% | FLOPs estimator: workload-aware | logical tokens/s: 67511.9 | valid tokens/s: 67511.9 | physical tokens/s: 68303.4 | loss tokens/s: 55283.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.740496E-01 | lm-global-supervised-tokens: 1.373827E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.148971E-02 | lm-token-buffer-mib: 2.360676E+02 | global_load_balancing_loss: 1.000625E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11153.692 | params rms: 0.060394 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:57:32.120352 | finish at 2026-10-17 22:23:03 + [2026-10-16 16:29:13] iteration 1951/ 2384 | consumed samples: 62432 | elapsed time per iteration (ms): 222496.5 | throughput per GPU (TFLOP/s/GPU): 25.9 | MFU 8.29% | FLOPs estimator: workload-aware | logical tokens/s: 75404.3 | valid tokens/s: 75404.3 | physical tokens/s: 75892.8 | loss tokens/s: 55481.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.681327E-01 | lm-global-supervised-tokens: 1.234433E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.202843E-02 | lm-token-buffer-mib: 2.329458E+02 | global_load_balancing_loss: 1.000087E+00 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11153.666 | params rms: 0.060393 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:45:40.982442 | finish at 2026-10-17 19:14:54 + [2026-10-16 16:32:53] iteration 1952/ 2384 | consumed samples: 62464 | elapsed time per iteration (ms): 220140.7 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.82% | FLOPs estimator: workload-aware | logical tokens/s: 76211.3 | valid tokens/s: 76211.3 | physical tokens/s: 77103.0 | loss tokens/s: 56085.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.061705E-01 | lm-global-supervised-tokens: 1.234660E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.931657E-02 | lm-token-buffer-mib: 2.354854E+02 | global_load_balancing_loss: 9.987323E-01 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11153.637 | params rms: 0.060393 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:25:00.775337 | finish at 2026-10-17 18:57:54 + [2026-10-16 16:36:38] iteration 1953/ 2384 | consumed samples: 62496 | elapsed time per iteration (ms): 224699.8 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.25% | FLOPs estimator: workload-aware | logical tokens/s: 74665.0 | valid tokens/s: 74665.0 | physical tokens/s: 75445.7 | loss tokens/s: 57160.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.233160E-01 | lm-global-supervised-tokens: 1.284384E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.484610E-02 | lm-token-buffer-mib: 2.343724E+02 | global_load_balancing_loss: 9.994987E-01 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11153.608 | params rms: 0.060393 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:54:05.612265 | finish at 2026-10-17 19:30:43 + [2026-10-16 16:40:18] iteration 1954/ 2384 | consumed samples: 62528 | elapsed time per iteration (ms): 220190.7 | throughput per GPU (TFLOP/s/GPU): 19.9 | MFU 6.37% | FLOPs estimator: workload-aware | logical tokens/s: 76194.0 | valid tokens/s: 76194.0 | physical tokens/s: 77095.4 | loss tokens/s: 59593.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.443402E-01 | lm-global-supervised-tokens: 1.312182E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.837821E-02 | lm-token-buffer-mib: 2.334017E+02 | global_load_balancing_loss: 1.000445E+00 | loss scale: 1.0 | grad norm: 0.141 | params norm: 11153.582 | params rms: 0.060393 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:18:01.995282 | finish at 2026-10-17 18:58:20 + [2026-10-16 16:43:59] iteration 1955/ 2384 | consumed samples: 62560 | elapsed time per iteration (ms): 220748.8 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.31% | FLOPs estimator: workload-aware | logical tokens/s: 76001.2 | valid tokens/s: 76001.2 | physical tokens/s: 77148.3 | loss tokens/s: 58523.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.000381E-01 | lm-global-supervised-tokens: 1.291896E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.307758E-02 | lm-token-buffer-mib: 2.325320E+02 | global_load_balancing_loss: 1.000583E+00 | loss scale: 1.0 | grad norm: 0.197 | params norm: 11153.555 | params rms: 0.060393 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:18:21.244422 | finish at 2026-10-17 19:02:20 + [2026-10-16 16:47:43] iteration 1956/ 2384 | consumed samples: 62592 | elapsed time per iteration (ms): 224225.4 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.31% | FLOPs estimator: workload-aware | logical tokens/s: 74822.8 | valid tokens/s: 74822.8 | physical tokens/s: 75761.5 | loss tokens/s: 60314.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.641169E-01 | lm-global-supervised-tokens: 1.352411E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.403802E-02 | lm-token-buffer-mib: 2.343831E+02 | global_load_balancing_loss: 1.001222E+00 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11153.530 | params rms: 0.060393 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:39:28.465983 | finish at 2026-10-17 19:27:11 + [2026-10-16 16:51:23] iteration 1957/ 2384 | consumed samples: 62624 | elapsed time per iteration (ms): 220405.9 | throughput per GPU (TFLOP/s/GPU): 25.1 | MFU 8.03% | FLOPs estimator: workload-aware | logical tokens/s: 76119.4 | valid tokens/s: 76119.4 | physical tokens/s: 76917.6 | loss tokens/s: 52665.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.958975E-01 | lm-global-supervised-tokens: 1.160780E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.440594E-02 | lm-token-buffer-mib: 2.341848E+02 | global_load_balancing_loss: 1.001120E+00 | loss scale: 1.0 | grad norm: 0.171 | params norm: 11153.501 | params rms: 0.060393 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:08:33.325409 | finish at 2026-10-17 18:59:57 + [2026-10-16 16:55:04] iteration 1958/ 2384 | consumed samples: 62656 | elapsed time per iteration (ms): 220331.1 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.85% | FLOPs estimator: workload-aware | logical tokens/s: 76145.5 | valid tokens/s: 76145.5 | physical tokens/s: 76747.6 | loss tokens/s: 57063.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.719452E-01 | lm-global-supervised-tokens: 1.257294E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.520390E-02 | lm-token-buffer-mib: 2.332434E+02 | global_load_balancing_loss: 1.000794E+00 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11153.470 | params rms: 0.060392 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:04:21.033258 | finish at 2026-10-17 18:59:25 + [2026-10-16 16:58:50] iteration 1959/ 2384 | consumed samples: 62688 | elapsed time per iteration (ms): 226485.4 | throughput per GPU (TFLOP/s/GPU): 25.0 | MFU 8.03% | FLOPs estimator: workload-aware | logical tokens/s: 74076.3 | valid tokens/s: 74076.3 | physical tokens/s: 74882.6 | loss tokens/s: 54596.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.679428E-01 | lm-global-supervised-tokens: 1.236536E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.228946E-02 | lm-token-buffer-mib: 2.365630E+02 | global_load_balancing_loss: 1.001430E+00 | loss scale: 1.0 | grad norm: 0.248 | params norm: 11153.441 | params rms: 0.060392 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:44:16.286877 | finish at 2026-10-17 19:43:06 + [2026-10-16 17:02:56] iteration 1960/ 2384 | consumed samples: 62720 | elapsed time per iteration (ms): 246084.3 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.20% | FLOPs estimator: workload-aware | logical tokens/s: 68176.6 | valid tokens/s: 68176.6 | physical tokens/s: 68522.8 | loss tokens/s: 49711.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.017786E-01 | lm-global-supervised-tokens: 1.223313E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.299146E-02 | lm-token-buffer-mib: 2.339022E+02 | global_load_balancing_loss: 1.000285E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11153.415 | params rms: 0.060392 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:58:59.758987 | finish at 2026-10-17 22:01:56 +times across ranks (ms): + save-checkpoint: + rank 0: 26029.44 + rank 1: 26029.46 + rank 2: 26029.45 + rank 3: 26029.46 + rank 4: 26029.40 + rank 5: 26029.46 + rank 6: 26029.43 + rank 7: 26029.44 + rank 8: 26028.87 + rank 9: 26028.88 + rank 10: 26028.89 + rank 11: 26028.86 + rank 12: 26028.88 + rank 13: 26028.85 + rank 14: 26028.87 + rank 15: 26029.24 + rank 16: 26030.11 + rank 17: 26030.11 + rank 18: 26029.95 + rank 19: 26030.12 + rank 20: 26030.10 + rank 21: 26030.09 + rank 22: 26029.97 + rank 23: 26030.13 + rank 24: 26029.01 + rank 25: 26029.00 + rank 26: 26029.01 + rank 27: 26028.96 + rank 28: 26029.28 + rank 29: 26029.11 + rank 30: 26028.89 + rank 31: 26028.96 + rank 32: 26029.21 + rank 33: 26029.22 + rank 34: 26029.27 + rank 35: 26029.25 + rank 36: 26029.29 + rank 37: 26029.20 + rank 38: 26029.29 + rank 39: 26029.31 + rank 40: 26028.50 + rank 41: 26028.49 + rank 42: 26028.51 + rank 43: 26028.51 + rank 44: 26028.48 + rank 45: 26028.50 + rank 46: 26028.48 + rank 47: 26028.48 + rank 48: 26029.06 + rank 49: 26028.95 + rank 50: 26028.95 + rank 51: 26029.09 + rank 52: 26028.89 + rank 53: 26029.16 + rank 54: 26029.01 + rank 55: 26029.02 + rank 56: 26028.82 + rank 57: 26028.87 + rank 58: 26028.85 + rank 59: 26028.87 + rank 60: 26028.89 + rank 61: 26028.84 + rank 62: 26028.82 + rank 63: 26028.83 + [2026-10-16 17:07:12] iteration 1961/ 2384 | consumed samples: 62752 | elapsed time per iteration (ms): 229058.2 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.26% | FLOPs estimator: workload-aware | logical tokens/s: 73244.3 | valid tokens/s: 73244.3 | physical tokens/s: 74014.4 | loss tokens/s: 54810.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.231524E-01 | lm-global-supervised-tokens: 1.255483E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 2.620668E-01 | lm-token-buffer-mib: 2.356098E+02 | global_load_balancing_loss: 1.000231E+00 | loss scale: 1.0 | grad norm: 0.598 | params norm: 11153.392 | params rms: 0.060392 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:54:51.623694 | finish at 2026-10-17 20:02:03 + [2026-10-16 17:10:59] iteration 1962/ 2384 | consumed samples: 62784 | elapsed time per iteration (ms): 227528.1 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.37% | FLOPs estimator: workload-aware | logical tokens/s: 73736.7 | valid tokens/s: 73736.7 | physical tokens/s: 74843.7 | loss tokens/s: 58247.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.888353E-01 | lm-global-supervised-tokens: 1.325289E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.013067E-02 | lm-token-buffer-mib: 2.342834E+02 | global_load_balancing_loss: 9.995705E-01 | loss scale: 1.0 | grad norm: 0.256 | params norm: 11153.364 | params rms: 0.060392 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:40:16.875109 | finish at 2026-10-17 19:51:16 + [2026-10-16 17:14:39] iteration 1963/ 2384 | consumed samples: 62816 | elapsed time per iteration (ms): 219764.8 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.54% | FLOPs estimator: workload-aware | logical tokens/s: 76341.4 | valid tokens/s: 76341.4 | physical tokens/s: 77161.2 | loss tokens/s: 57700.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.622435E-01 | lm-global-supervised-tokens: 1.268062E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.558385E-02 | lm-token-buffer-mib: 2.336555E+02 | global_load_balancing_loss: 9.995968E-01 | loss scale: 1.0 | grad norm: 0.885 | params norm: 11153.333 | params rms: 0.060392 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:42:00.976313 | finish at 2026-10-17 18:56:40 + [2026-10-16 17:18:25] iteration 1964/ 2384 | consumed samples: 62848 | elapsed time per iteration (ms): 225750.1 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.43% | FLOPs estimator: workload-aware | logical tokens/s: 74317.4 | valid tokens/s: 74317.4 | physical tokens/s: 75819.3 | loss tokens/s: 57374.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.310892E-01 | lm-global-supervised-tokens: 1.295229E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.866677E-02 | lm-token-buffer-mib: 2.373873E+02 | global_load_balancing_loss: 9.987738E-01 | loss scale: 1.0 | grad norm: 0.247 | params norm: 11153.300 | params rms: 0.060391 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:20:15.054073 | finish at 2026-10-17 19:38:40 + [2026-10-16 17:22:06] iteration 1965/ 2384 | consumed samples: 62880 | elapsed time per iteration (ms): 221932.8 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.61% | FLOPs estimator: workload-aware | logical tokens/s: 75595.8 | valid tokens/s: 75595.8 | physical tokens/s: 76359.6 | loss tokens/s: 56733.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.581483E-01 | lm-global-supervised-tokens: 1.259094E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.245000E-02 | lm-token-buffer-mib: 2.357923E+02 | global_load_balancing_loss: 1.000985E+00 | loss scale: 1.0 | grad norm: 0.778 | params norm: 11153.266 | params rms: 0.060391 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:49:49.836630 | finish at 2026-10-17 19:11:56 + [2026-10-16 17:25:48] iteration 1966/ 2384 | consumed samples: 62912 | elapsed time per iteration (ms): 221634.8 | throughput per GPU (TFLOP/s/GPU): 24.5 | MFU 7.85% | FLOPs estimator: workload-aware | logical tokens/s: 75697.5 | valid tokens/s: 75697.5 | physical tokens/s: 76297.3 | loss tokens/s: 54980.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.736749E-01 | lm-global-supervised-tokens: 1.218564E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.212390E-02 | lm-token-buffer-mib: 2.349360E+02 | global_load_balancing_loss: 1.002273E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11153.231 | params rms: 0.060391 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:44:03.337114 | finish at 2026-10-17 19:09:51 + [2026-10-16 17:29:31] iteration 1967/ 2384 | consumed samples: 62944 | elapsed time per iteration (ms): 223102.4 | throughput per GPU (TFLOP/s/GPU): 24.9 | MFU 7.97% | FLOPs estimator: workload-aware | logical tokens/s: 75199.5 | valid tokens/s: 75199.5 | physical tokens/s: 75682.7 | loss tokens/s: 54744.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.826199E-01 | lm-global-supervised-tokens: 1.221357E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.954193E-02 | lm-token-buffer-mib: 2.344044E+02 | global_load_balancing_loss: 1.002149E+00 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11153.195 | params rms: 0.060391 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:50:33.701722 | finish at 2026-10-17 19:20:05 + [2026-10-16 17:33:14] iteration 1968/ 2384 | consumed samples: 62976 | elapsed time per iteration (ms): 223052.4 | throughput per GPU (TFLOP/s/GPU): 18.6 | MFU 5.96% | FLOPs estimator: workload-aware | logical tokens/s: 75216.3 | valid tokens/s: 75216.3 | physical tokens/s: 76212.7 | loss tokens/s: 57863.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.286983E-01 | lm-global-supervised-tokens: 1.290669E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.509462E-02 | lm-token-buffer-mib: 2.352338E+02 | global_load_balancing_loss: 1.001059E+00 | loss scale: 1.0 | grad norm: 0.174 | params norm: 11153.161 | params rms: 0.060391 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:46:29.805389 | finish at 2026-10-17 19:19:44 + [2026-10-16 17:36:59] iteration 1969/ 2384 | consumed samples: 63008 | elapsed time per iteration (ms): 225028.9 | throughput per GPU (TFLOP/s/GPU): 25.1 | MFU 8.03% | FLOPs estimator: workload-aware | logical tokens/s: 74555.7 | valid tokens/s: 74555.7 | physical tokens/s: 75479.6 | loss tokens/s: 57746.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.435941E-01 | lm-global-supervised-tokens: 1.299453E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.312051E-02 | lm-token-buffer-mib: 2.353009E+02 | global_load_balancing_loss: 1.000605E+00 | loss scale: 1.0 | grad norm: 0.428 | params norm: 11153.129 | params rms: 0.060391 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:56:26.990494 | finish at 2026-10-17 19:33:26 + [2026-10-16 17:41:06] iteration 1970/ 2384 | consumed samples: 63040 | elapsed time per iteration (ms): 246551.3 | throughput per GPU (TFLOP/s/GPU): 19.2 | MFU 6.14% | FLOPs estimator: workload-aware | logical tokens/s: 68047.4 | valid tokens/s: 68047.4 | physical tokens/s: 68377.0 | loss tokens/s: 48897.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.802209E-01 | lm-global-supervised-tokens: 1.205569E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.796364E-02 | lm-token-buffer-mib: 2.328160E+02 | global_load_balancing_loss: 1.000587E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11153.096 | params rms: 0.060390 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:21:12.221243 | finish at 2026-10-17 22:02:18 + [2026-10-16 17:44:46] iteration 1971/ 2384 | consumed samples: 63072 | elapsed time per iteration (ms): 220422.8 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.79% | FLOPs estimator: workload-aware | logical tokens/s: 76113.7 | valid tokens/s: 76113.7 | physical tokens/s: 76479.1 | loss tokens/s: 58211.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.372039E-01 | lm-global-supervised-tokens: 1.283120E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.568643E-02 | lm-token-buffer-mib: 2.336850E+02 | global_load_balancing_loss: 9.986445E-01 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11153.060 | params rms: 0.060390 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:17:14.604118 | finish at 2026-10-17 19:02:01 + [2026-10-16 17:48:31] iteration 1972/ 2384 | consumed samples: 63104 | elapsed time per iteration (ms): 224371.2 | throughput per GPU (TFLOP/s/GPU): 17.6 | MFU 5.65% | FLOPs estimator: workload-aware | logical tokens/s: 74774.3 | valid tokens/s: 74774.3 | physical tokens/s: 75380.2 | loss tokens/s: 58589.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.426507E-01 | lm-global-supervised-tokens: 1.314585E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.860877E-02 | lm-token-buffer-mib: 2.341961E+02 | global_load_balancing_loss: 9.988567E-01 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11153.023 | params rms: 0.060390 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:40:40.926085 | finish at 2026-10-17 19:29:12 + [2026-10-16 17:52:15] iteration 1973/ 2384 | consumed samples: 63136 | elapsed time per iteration (ms): 224694.4 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.02% | FLOPs estimator: workload-aware | logical tokens/s: 74666.7 | valid tokens/s: 74666.7 | physical tokens/s: 75769.4 | loss tokens/s: 55948.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.161729E-01 | lm-global-supervised-tokens: 1.257133E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 5.310518E-02 | lm-token-buffer-mib: 2.340830E+02 | global_load_balancing_loss: 1.000805E+00 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11152.989 | params rms: 0.060390 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:39:09.405877 | finish at 2026-10-17 19:31:25 + [2026-10-16 17:56:00] iteration 1974/ 2384 | consumed samples: 63168 | elapsed time per iteration (ms): 224437.8 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.36% | FLOPs estimator: workload-aware | logical tokens/s: 74752.1 | valid tokens/s: 74752.1 | physical tokens/s: 75158.8 | loss tokens/s: 54755.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.794890E-01 | lm-global-supervised-tokens: 1.228923E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.379695E-02 | lm-token-buffer-mib: 2.331137E+02 | global_load_balancing_loss: 1.000219E+00 | loss scale: 1.0 | grad norm: 0.121 | params norm: 11152.958 | params rms: 0.060390 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:33:39.518206 | finish at 2026-10-17 19:29:39 + [2026-10-16 17:59:42] iteration 1975/ 2384 | consumed samples: 63200 | elapsed time per iteration (ms): 222344.8 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.89% | FLOPs estimator: workload-aware | logical tokens/s: 75455.7 | valid tokens/s: 75455.7 | physical tokens/s: 76126.0 | loss tokens/s: 59250.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.119865E-01 | lm-global-supervised-tokens: 1.317404E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.001161E-02 | lm-token-buffer-mib: 2.329081E+02 | global_load_balancing_loss: 1.002158E+00 | loss scale: 1.0 | grad norm: 0.200 | params norm: 11152.926 | params rms: 0.060389 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:15:39.007889 | finish at 2026-10-17 19:15:21 + [2026-10-16 18:03:25] iteration 1976/ 2384 | consumed samples: 63232 | elapsed time per iteration (ms): 223001.5 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.78% | FLOPs estimator: workload-aware | logical tokens/s: 75233.6 | valid tokens/s: 75233.6 | physical tokens/s: 76048.1 | loss tokens/s: 57798.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.815625E-01 | lm-global-supervised-tokens: 1.288915E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.173237E-02 | lm-token-buffer-mib: 2.354229E+02 | global_load_balancing_loss: 1.000169E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11152.892 | params rms: 0.060389 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:16:24.618668 | finish at 2026-10-17 19:19:50 + [2026-10-16 18:07:04] iteration 1977/ 2384 | consumed samples: 63264 | elapsed time per iteration (ms): 219126.9 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.62% | FLOPs estimator: workload-aware | logical tokens/s: 76563.8 | valid tokens/s: 76563.8 | physical tokens/s: 77625.3 | loss tokens/s: 60896.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.297991E-01 | lm-global-supervised-tokens: 1.334409E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.394846E-02 | lm-token-buffer-mib: 2.336574E+02 | global_load_balancing_loss: 1.001352E+00 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11152.862 | params rms: 0.060389 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:46:24.661771 | finish at 2026-10-17 18:53:29 + [2026-10-16 18:10:44] iteration 1978/ 2384 | consumed samples: 63296 | elapsed time per iteration (ms): 219438.9 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.66% | FLOPs estimator: workload-aware | logical tokens/s: 76454.9 | valid tokens/s: 76454.9 | physical tokens/s: 76957.6 | loss tokens/s: 58195.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.330319E-01 | lm-global-supervised-tokens: 1.277030E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.736134E-02 | lm-token-buffer-mib: 2.336792E+02 | global_load_balancing_loss: 1.001149E+00 | loss scale: 1.0 | grad norm: 0.105 | params norm: 11152.835 | params rms: 0.060389 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:44:52.193978 | finish at 2026-10-17 18:55:36 + [2026-10-16 18:14:26] iteration 1979/ 2384 | consumed samples: 63328 | elapsed time per iteration (ms): 222491.1 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.05% | FLOPs estimator: workload-aware | logical tokens/s: 75406.1 | valid tokens/s: 75406.1 | physical tokens/s: 76509.1 | loss tokens/s: 58557.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.709150E-01 | lm-global-supervised-tokens: 1.302860E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.334299E-02 | lm-token-buffer-mib: 2.334068E+02 | global_load_balancing_loss: 1.000827E+00 | loss scale: 1.0 | grad norm: 0.155 | params norm: 11152.802 | params rms: 0.060389 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:01:48.883846 | finish at 2026-10-17 19:16:15 + [2026-10-16 18:18:29] iteration 1980/ 2384 | consumed samples: 63360 | elapsed time per iteration (ms): 242381.3 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.99% | FLOPs estimator: workload-aware | logical tokens/s: 69218.2 | valid tokens/s: 69218.2 | physical tokens/s: 69897.7 | loss tokens/s: 51083.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.561463E-01 | lm-global-supervised-tokens: 1.238157E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.883774E-02 | lm-token-buffer-mib: 2.354897E+02 | global_load_balancing_loss: 1.000204E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11152.767 | params rms: 0.060389 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:12:02.035557 | finish at 2026-10-17 21:30:31 + [2026-10-16 18:22:10] iteration 1981/ 2384 | consumed samples: 63392 | elapsed time per iteration (ms): 221237.9 | throughput per GPU (TFLOP/s/GPU): 24.5 | MFU 7.84% | FLOPs estimator: workload-aware | logical tokens/s: 75833.3 | valid tokens/s: 75833.3 | physical tokens/s: 76157.1 | loss tokens/s: 54863.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.003700E-01 | lm-global-supervised-tokens: 1.213798E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.282500E-02 | lm-token-buffer-mib: 2.331562E+02 | global_load_balancing_loss: 1.000688E+00 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11152.732 | params rms: 0.060388 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:45:58.870345 | finish at 2026-10-17 19:08:09 + [2026-10-16 18:25:55] iteration 1982/ 2384 | consumed samples: 63424 | elapsed time per iteration (ms): 224777.7 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.32% | FLOPs estimator: workload-aware | logical tokens/s: 74639.1 | valid tokens/s: 74639.1 | physical tokens/s: 75402.5 | loss tokens/s: 56243.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.912289E-01 | lm-global-supervised-tokens: 1.264226E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.480719E-02 | lm-token-buffer-mib: 2.359669E+02 | global_load_balancing_loss: 1.000673E+00 | loss scale: 1.0 | grad norm: 0.117 | params norm: 11152.699 | params rms: 0.060388 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:06:00.652631 | finish at 2026-10-17 19:31:55 + [2026-10-16 18:29:36] iteration 1983/ 2384 | consumed samples: 63456 | elapsed time per iteration (ms): 221055.8 | throughput per GPU (TFLOP/s/GPU): 19.9 | MFU 6.39% | FLOPs estimator: workload-aware | logical tokens/s: 75895.7 | valid tokens/s: 75895.7 | physical tokens/s: 76504.4 | loss tokens/s: 58117.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.409704E-01 | lm-global-supervised-tokens: 1.284712E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.589386E-02 | lm-token-buffer-mib: 2.344433E+02 | global_load_balancing_loss: 1.000428E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11152.670 | params rms: 0.060388 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:37:23.377601 | finish at 2026-10-17 19:06:59 + [2026-10-16 18:33:18] iteration 1984/ 2384 | consumed samples: 63488 | elapsed time per iteration (ms): 222617.9 | throughput per GPU (TFLOP/s/GPU): 24.1 | MFU 7.73% | FLOPs estimator: workload-aware | logical tokens/s: 75363.0 | valid tokens/s: 75363.0 | physical tokens/s: 76203.3 | loss tokens/s: 55877.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.040363E-01 | lm-global-supervised-tokens: 1.243931E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.270684E-02 | lm-token-buffer-mib: 2.332142E+02 | global_load_balancing_loss: 1.001482E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11152.642 | params rms: 0.060388 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:44:07.167492 | finish at 2026-10-17 19:17:25 + [2026-10-16 18:37:04] iteration 1985/ 2384 | consumed samples: 63520 | elapsed time per iteration (ms): 225364.0 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.54% | FLOPs estimator: workload-aware | logical tokens/s: 74444.9 | valid tokens/s: 74444.9 | physical tokens/s: 74678.7 | loss tokens/s: 55278.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.509659E-01 | lm-global-supervised-tokens: 1.245768E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.364035E-02 | lm-token-buffer-mib: 2.324381E+02 | global_load_balancing_loss: 1.000098E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11152.617 | params rms: 0.060388 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:58:40.218339 | finish at 2026-10-17 19:35:44 + [2026-10-16 18:40:50] iteration 1986/ 2384 | consumed samples: 63552 | elapsed time per iteration (ms): 226212.3 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.46% | FLOPs estimator: workload-aware | logical tokens/s: 74165.6 | valid tokens/s: 74165.6 | physical tokens/s: 74961.7 | loss tokens/s: 55377.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.595322E-01 | lm-global-supervised-tokens: 1.252701E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.520459E-02 | lm-token-buffer-mib: 2.348432E+02 | global_load_balancing_loss: 9.999644E-01 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11152.592 | params rms: 0.060388 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:00:32.492863 | finish at 2026-10-17 19:41:22 + [2026-10-16 18:44:35] iteration 1987/ 2384 | consumed samples: 63584 | elapsed time per iteration (ms): 224784.6 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.57% | FLOPs estimator: workload-aware | logical tokens/s: 74636.7 | valid tokens/s: 74636.7 | physical tokens/s: 76222.2 | loss tokens/s: 54016.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.958098E-01 | lm-global-supervised-tokens: 1.214196E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.722876E-02 | lm-token-buffer-mib: 2.340011E+02 | global_load_balancing_loss: 1.000315E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11152.571 | params rms: 0.060388 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:47:19.473808 | finish at 2026-10-17 19:31:54 + [2026-10-16 18:48:21] iteration 1988/ 2384 | consumed samples: 63616 | elapsed time per iteration (ms): 226234.5 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.67% | FLOPs estimator: workload-aware | logical tokens/s: 74158.5 | valid tokens/s: 74158.5 | physical tokens/s: 75767.3 | loss tokens/s: 56158.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.130351E-01 | lm-global-supervised-tokens: 1.270493E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.244659E-02 | lm-token-buffer-mib: 2.363185E+02 | global_load_balancing_loss: 9.993272E-01 | loss scale: 1.0 | grad norm: 0.160 | params norm: 11152.548 | params rms: 0.060387 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:53:08.880572 | finish at 2026-10-17 19:41:30 + [2026-10-16 18:52:02] iteration 1989/ 2384 | consumed samples: 63648 | elapsed time per iteration (ms): 221434.0 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.08% | FLOPs estimator: workload-aware | logical tokens/s: 75766.0 | valid tokens/s: 75766.0 | physical tokens/s: 76198.0 | loss tokens/s: 57145.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.230700E-01 | lm-global-supervised-tokens: 1.265405E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.766756E-02 | lm-token-buffer-mib: 2.336480E+02 | global_load_balancing_loss: 9.995440E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11152.525 | params rms: 0.060387 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:17:46.440177 | finish at 2026-10-17 19:09:49 + [2026-10-16 18:56:07] iteration 1990/ 2384 | consumed samples: 63680 | elapsed time per iteration (ms): 244625.0 | throughput per GPU (TFLOP/s/GPU): 18.6 | MFU 5.98% | FLOPs estimator: workload-aware | logical tokens/s: 68583.3 | valid tokens/s: 68583.3 | physical tokens/s: 69620.0 | loss tokens/s: 52226.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.887054E-01 | lm-global-supervised-tokens: 1.277602E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.379446E-02 | lm-token-buffer-mib: 2.365483E+02 | global_load_balancing_loss: 1.000820E+00 | loss scale: 1.0 | grad norm: 0.111 | params norm: 11152.504 | params rms: 0.060387 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:46:22.251785 | finish at 2026-10-17 21:42:29 + [2026-10-16 18:59:48] iteration 1991/ 2384 | consumed samples: 63712 | elapsed time per iteration (ms): 220724.7 | throughput per GPU (TFLOP/s/GPU): 19.7 | MFU 6.30% | FLOPs estimator: workload-aware | logical tokens/s: 76009.5 | valid tokens/s: 76009.5 | physical tokens/s: 77077.0 | loss tokens/s: 63758.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.147884E-01 | lm-global-supervised-tokens: 1.407302E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.517647E-02 | lm-token-buffer-mib: 2.344137E+02 | global_load_balancing_loss: 1.000695E+00 | loss scale: 1.0 | grad norm: 0.196 | params norm: 11152.484 | params rms: 0.060387 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:05:44.823019 | finish at 2026-10-17 19:05:32 + [2026-10-16 19:03:30] iteration 1992/ 2384 | consumed samples: 63744 | elapsed time per iteration (ms): 222121.7 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.58% | FLOPs estimator: workload-aware | logical tokens/s: 75531.6 | valid tokens/s: 75531.6 | physical tokens/s: 76661.1 | loss tokens/s: 56501.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.418422E-01 | lm-global-supervised-tokens: 1.255013E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.028666E-02 | lm-token-buffer-mib: 2.350262E+02 | global_load_balancing_loss: 1.000689E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11152.462 | params rms: 0.060387 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:11:11.695484 | finish at 2026-10-17 19:14:41 + [2026-10-16 19:07:11] iteration 1993/ 2384 | consumed samples: 63776 | elapsed time per iteration (ms): 221526.0 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.35% | FLOPs estimator: workload-aware | logical tokens/s: 75734.7 | valid tokens/s: 75734.7 | physical tokens/s: 76446.9 | loss tokens/s: 57604.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.695890E-01 | lm-global-supervised-tokens: 1.276084E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.304574E-02 | lm-token-buffer-mib: 2.334383E+02 | global_load_balancing_loss: 1.001720E+00 | loss scale: 1.0 | grad norm: 0.218 | params norm: 11152.440 | params rms: 0.060387 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:03:36.664051 | finish at 2026-10-17 19:10:48 + [2026-10-16 19:10:51] iteration 1994/ 2384 | consumed samples: 63808 | elapsed time per iteration (ms): 219720.0 | throughput per GPU (TFLOP/s/GPU): 25.7 | MFU 8.24% | FLOPs estimator: workload-aware | logical tokens/s: 76357.1 | valid tokens/s: 76357.1 | physical tokens/s: 76659.3 | loss tokens/s: 59510.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.870210E-01 | lm-global-supervised-tokens: 1.307562E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.509494E-02 | lm-token-buffer-mib: 2.330138E+02 | global_load_balancing_loss: 1.001748E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11152.416 | params rms: 0.060387 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:48:10.810239 | finish at 2026-10-17 18:59:02 + [2026-10-16 19:14:36] iteration 1995/ 2384 | consumed samples: 63840 | elapsed time per iteration (ms): 224988.9 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.44% | FLOPs estimator: workload-aware | logical tokens/s: 74568.9 | valid tokens/s: 74568.9 | physical tokens/s: 75936.8 | loss tokens/s: 59838.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.645096E-01 | lm-global-supervised-tokens: 1.346297E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.741372E-02 | lm-token-buffer-mib: 2.382797E+02 | global_load_balancing_loss: 1.000608E+00 | loss scale: 1.0 | grad norm: 0.192 | params norm: 11152.390 | params rms: 0.060387 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:18:40.673639 | finish at 2026-10-17 19:33:17 + [2026-10-16 19:18:18] iteration 1996/ 2384 | consumed samples: 63872 | elapsed time per iteration (ms): 221536.5 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.85% | FLOPs estimator: workload-aware | logical tokens/s: 75731.1 | valid tokens/s: 75731.1 | physical tokens/s: 76151.8 | loss tokens/s: 57081.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.838608E-01 | lm-global-supervised-tokens: 1.264573E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.142196E-02 | lm-token-buffer-mib: 2.338685E+02 | global_load_balancing_loss: 9.982662E-01 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11152.366 | params rms: 0.060386 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:52:36.168929 | finish at 2026-10-17 19:10:54 + [2026-10-16 19:22:01] iteration 1997/ 2384 | consumed samples: 63904 | elapsed time per iteration (ms): 223074.3 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.01% | FLOPs estimator: workload-aware | logical tokens/s: 75209.0 | valid tokens/s: 75209.0 | physical tokens/s: 75942.1 | loss tokens/s: 55849.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.702866E-01 | lm-global-supervised-tokens: 1.245866E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.480974E-02 | lm-token-buffer-mib: 2.333261E+02 | global_load_balancing_loss: 9.995025E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11152.340 | params rms: 0.060386 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:58:49.770728 | finish at 2026-10-17 19:20:50 + [2026-10-16 19:25:42] iteration 1998/ 2384 | consumed samples: 63936 | elapsed time per iteration (ms): 220901.4 | throughput per GPU (TFLOP/s/GPU): 26.0 | MFU 8.33% | FLOPs estimator: workload-aware | logical tokens/s: 75948.7 | valid tokens/s: 75948.7 | physical tokens/s: 76658.4 | loss tokens/s: 56033.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.461400E-01 | lm-global-supervised-tokens: 1.237787E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.653079E-02 | lm-token-buffer-mib: 2.344470E+02 | global_load_balancing_loss: 9.992684E-01 | loss scale: 1.0 | grad norm: 0.141 | params norm: 11152.313 | params rms: 0.060386 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:41:07.949177 | finish at 2026-10-17 19:06:49 + [2026-10-16 19:29:27] iteration 1999/ 2384 | consumed samples: 63968 | elapsed time per iteration (ms): 225952.2 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.72% | FLOPs estimator: workload-aware | logical tokens/s: 74251.0 | valid tokens/s: 74251.0 | physical tokens/s: 74953.3 | loss tokens/s: 54759.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.242782E-01 | lm-global-supervised-tokens: 1.237296E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.117684E-02 | lm-token-buffer-mib: 2.342367E+02 | global_load_balancing_loss: 1.000865E+00 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11152.284 | params rms: 0.060386 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:09:51.615517 | finish at 2026-10-17 19:39:19 + [2026-10-16 19:33:33] iteration 2000/ 2384 | consumed samples: 64000 | elapsed time per iteration (ms): 245985.5 | throughput per GPU (TFLOP/s/GPU): 17.9 | MFU 5.73% | FLOPs estimator: workload-aware | logical tokens/s: 68204.1 | valid tokens/s: 68204.1 | physical tokens/s: 68816.6 | loss tokens/s: 50834.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.717943E-01 | lm-global-supervised-tokens: 1.250461E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.389248E-02 | lm-token-buffer-mib: 2.342782E+02 | global_load_balancing_loss: 9.997483E-01 | loss scale: 1.0 | grad norm: 0.149 | params norm: 11152.254 | params rms: 0.060386 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:14:18.417572 | finish at 2026-10-17 21:47:52 +times across ranks (ms): + save-checkpoint: + rank 0: 25574.64 + rank 1: 25574.65 + rank 2: 25574.62 + rank 3: 25574.72 + rank 4: 25574.62 + rank 5: 25574.67 + rank 6: 25574.71 + rank 7: 25574.61 + rank 8: 25573.92 + rank 9: 25573.89 + rank 10: 25573.89 + rank 11: 25573.94 + rank 12: 25573.87 + rank 13: 25573.98 + rank 14: 25573.91 + rank 15: 25573.83 + rank 16: 25573.93 + rank 17: 25573.92 + rank 18: 25573.93 + rank 19: 25573.92 + rank 20: 25573.91 + rank 21: 25573.91 + rank 22: 25573.92 + rank 23: 25573.91 + rank 24: 25575.47 + rank 25: 25575.50 + rank 26: 25575.47 + rank 27: 25575.49 + rank 28: 25575.35 + rank 29: 25575.87 + rank 30: 25575.46 + rank 31: 25575.47 + rank 32: 25572.12 + rank 33: 25572.07 + rank 34: 25572.18 + rank 35: 25572.14 + rank 36: 25572.04 + rank 37: 25572.15 + rank 38: 25572.00 + rank 39: 25572.00 + rank 40: 25574.01 + rank 41: 25573.96 + rank 42: 25573.96 + rank 43: 25573.98 + rank 44: 25573.94 + rank 45: 25573.94 + rank 46: 25573.93 + rank 47: 25573.95 + rank 48: 25573.51 + rank 49: 25573.48 + rank 50: 25573.55 + rank 51: 25573.58 + rank 52: 25573.46 + rank 53: 25573.56 + rank 54: 25573.47 + rank 55: 25573.52 + rank 56: 25574.04 + rank 57: 25574.14 + rank 58: 25574.14 + rank 59: 25574.09 + rank 60: 25574.09 + rank 61: 25573.90 + rank 62: 25574.05 + rank 63: 25574.06 + [2026-10-16 19:37:55] iteration 2001/ 2384 | consumed samples: 64032 | elapsed time per iteration (ms): 236188.0 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.84% | FLOPs estimator: workload-aware | logical tokens/s: 71033.2 | valid tokens/s: 71033.2 | physical tokens/s: 71554.9 | loss tokens/s: 45252.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.857687E-01 | lm-global-supervised-tokens: 1.068818E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.205349E-01 | lm-token-buffer-mib: 2.352822E+02 | global_load_balancing_loss: 1.000858E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11152.225 | params rms: 0.060386 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:07:40.010013 | finish at 2026-10-17 20:45:35 +times across ranks (ms): + forward-backward: + rank 0: 235591.45 + rank 1: 235587.08 + rank 2: 235587.72 + rank 3: 235589.25 + rank 4: 235595.11 + rank 5: 235587.97 + rank 6: 235590.11 + rank 7: 235590.42 + rank 8: 235994.78 + rank 9: 235991.91 + rank 10: 235989.59 + rank 11: 235992.88 + rank 12: 235997.64 + rank 13: 235993.83 + rank 14: 235991.28 + rank 15: 235997.58 + rank 16: 235261.91 + rank 17: 235264.58 + rank 18: 235258.67 + rank 19: 235265.56 + rank 20: 235262.97 + rank 21: 235261.56 + rank 22: 235257.55 + rank 23: 235264.92 + rank 24: 235711.14 + rank 25: 235705.70 + rank 26: 235703.58 + rank 27: 235704.53 + rank 28: 235708.83 + rank 29: 235708.39 + rank 30: 235706.23 + rank 31: 235711.50 + rank 32: 235550.09 + rank 33: 235545.48 + rank 34: 235545.03 + rank 35: 235545.89 + rank 36: 235553.59 + rank 37: 235545.59 + rank 38: 235549.36 + rank 39: 235550.67 + rank 40: 235985.50 + rank 41: 235979.50 + rank 42: 235977.17 + rank 43: 235980.36 + rank 44: 235982.28 + rank 45: 235976.34 + rank 46: 235983.23 + rank 47: 235986.69 + rank 48: 235262.98 + rank 49: 235264.11 + rank 50: 235256.81 + rank 51: 235264.03 + rank 52: 235259.00 + rank 53: 235258.47 + rank 54: 235260.59 + rank 55: 235266.62 + rank 56: 235713.44 + rank 57: 235707.52 + rank 58: 235703.33 + rank 59: 235704.80 + rank 60: 235703.58 + rank 61: 235703.19 + rank 62: 235710.94 + rank 63: 235714.03 + forward-compute: + rank 0: 130949.71 + rank 1: 130940.59 + rank 2: 130930.24 + rank 3: 130924.23 + rank 4: 130936.09 + rank 5: 130945.93 + rank 6: 130944.07 + rank 7: 130951.23 + rank 8: 131360.72 + rank 9: 131361.02 + rank 10: 131351.91 + rank 11: 131367.17 + rank 12: 131390.77 + rank 13: 131375.28 + rank 14: 131356.23 + rank 15: 131370.81 + rank 16: 130602.50 + rank 17: 130600.27 + rank 18: 130594.08 + rank 19: 130602.88 + rank 20: 130599.62 + rank 21: 130609.51 + rank 22: 130608.38 + rank 23: 130613.18 + rank 24: 131347.30 + rank 25: 131349.34 + rank 26: 131749.78 + rank 27: 131576.00 + rank 28: 131358.27 + rank 29: 131599.56 + rank 30: 131396.31 + rank 31: 131684.66 + rank 32: 74356.23 + rank 33: 74343.27 + rank 34: 74347.02 + rank 35: 74354.91 + rank 36: 74369.98 + rank 37: 74390.70 + rank 38: 74358.26 + rank 39: 74369.74 + rank 40: 74831.92 + rank 41: 74792.77 + rank 42: 74797.05 + rank 43: 74810.97 + rank 44: 74796.94 + rank 45: 74800.75 + rank 46: 74767.56 + rank 47: 74817.92 + rank 48: 74049.98 + rank 49: 74053.02 + rank 50: 74059.63 + rank 51: 74067.02 + rank 52: 74093.05 + rank 53: 74084.70 + rank 54: 74040.34 + rank 55: 74070.24 + rank 56: 74503.58 + rank 57: 74484.06 + rank 58: 74475.55 + rank 59: 74470.29 + rank 60: 74490.30 + rank 61: 74518.98 + rank 62: 74474.37 + rank 63: 74512.03 + backward-compute: + rank 0: 104594.59 + rank 1: 104584.73 + rank 2: 104606.65 + rank 3: 104615.88 + rank 4: 104600.46 + rank 5: 104593.02 + rank 6: 104590.71 + rank 7: 104592.87 + rank 8: 104585.55 + rank 9: 104578.73 + rank 10: 104584.97 + rank 11: 104574.95 + rank 12: 104555.27 + rank 13: 104566.50 + rank 14: 104582.08 + rank 15: 104576.85 + rank 16: 104618.73 + rank 17: 104614.53 + rank 18: 104615.05 + rank 19: 104610.50 + rank 20: 104612.98 + rank 21: 104604.57 + rank 22: 104600.35 + rank 23: 104606.74 + rank 24: 104305.66 + rank 25: 104298.05 + rank 26: 103890.01 + rank 27: 104068.66 + rank 28: 104281.76 + rank 29: 104040.59 + rank 30: 104251.52 + rank 31: 103962.24 + rank 32: 160985.22 + rank 33: 160992.08 + rank 34: 160988.55 + rank 35: 160980.34 + rank 36: 160974.12 + rank 37: 160944.69 + rank 38: 160980.38 + rank 39: 160971.27 + rank 40: 161118.14 + rank 41: 161150.77 + rank 42: 161145.31 + rank 43: 161134.00 + rank 44: 161150.89 + rank 45: 161140.92 + rank 46: 161179.50 + rank 47: 161135.03 + rank 48: 161180.14 + rank 49: 161176.20 + rank 50: 161159.59 + rank 51: 161160.00 + rank 52: 161133.91 + rank 53: 161137.12 + rank 54: 161185.28 + rank 55: 161161.38 + rank 56: 161174.67 + rank 57: 161183.80 + rank 58: 161191.70 + rank 59: 161195.59 + rank 60: 161177.48 + rank 61: 161146.77 + rank 62: 161199.78 + rank 63: 161164.52 + batch-generator: + rank 0: 34.55 + rank 1: 35.71 + rank 2: 36.50 + rank 3: 35.71 + rank 4: 37.90 + rank 5: 36.29 + rank 6: 35.89 + rank 7: 33.47 + rank 8: 42.11 + rank 9: 47.45 + rank 10: 43.05 + rank 11: 47.88 + rank 12: 43.53 + rank 13: 43.76 + rank 14: 44.56 + rank 15: 42.28 + rank 16: 32.23 + rank 17: 35.17 + rank 18: 37.30 + rank 19: 36.66 + rank 20: 36.09 + rank 21: 36.38 + rank 22: 37.37 + rank 23: 32.56 + rank 24: 45.03 + rank 25: 46.35 + rank 26: 50.38 + rank 27: 49.45 + rank 28: 47.11 + rank 29: 47.68 + rank 30: 45.42 + rank 31: 44.47 + rank 32: 46.02 + rank 33: 51.96 + rank 34: 48.82 + rank 35: 50.61 + rank 36: 46.78 + rank 37: 47.32 + rank 38: 46.36 + rank 39: 41.67 + rank 40: 35.02 + rank 41: 37.74 + rank 42: 36.56 + rank 43: 38.37 + rank 44: 37.44 + rank 45: 36.31 + rank 46: 36.94 + rank 47: 32.92 + rank 48: 40.59 + rank 49: 42.98 + rank 50: 44.76 + rank 51: 45.58 + rank 52: 33.71 + rank 53: 42.58 + rank 54: 36.14 + rank 55: 38.50 + rank 56: 32.82 + rank 57: 38.94 + rank 58: 39.34 + rank 59: 39.57 + rank 60: 42.04 + rank 61: 37.25 + rank 62: 38.28 + rank 63: 33.95 + embedding-grads-all-reduce: + rank 0: 0.03 + rank 1: 0.03 + rank 2: 0.03 + rank 3: 0.04 + rank 4: 0.03 + rank 5: 0.03 + rank 6: 0.03 + rank 7: 0.03 + rank 8: 0.03 + rank 9: 0.03 + rank 10: 0.04 + rank 11: 0.03 + rank 12: 0.03 + rank 13: 0.03 + rank 14: 0.03 + rank 15: 0.03 + rank 16: 0.03 + rank 17: 0.03 + rank 18: 0.03 + rank 19: 0.03 + rank 20: 0.03 + rank 21: 0.03 + rank 22: 0.03 + rank 23: 0.04 + rank 24: 0.04 + rank 25: 0.04 + rank 26: 0.04 + rank 27: 0.04 + rank 28: 0.04 + rank 29: 0.04 + rank 30: 0.04 + rank 31: 0.04 + rank 32: 0.03 + rank 33: 0.04 + rank 34: 0.03 + rank 35: 0.04 + rank 36: 0.03 + rank 37: 0.04 + rank 38: 0.03 + rank 39: 0.04 + rank 40: 0.03 + rank 41: 0.03 + rank 42: 0.03 + rank 43: 0.03 + rank 44: 0.03 + rank 45: 0.03 + rank 46: 0.03 + rank 47: 0.03 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.03 + rank 57: 0.03 + rank 58: 0.03 + rank 59: 0.03 + rank 60: 0.03 + rank 61: 0.03 + rank 62: 0.03 + rank 63: 0.03 + all-grads-sync: + rank 0: 0.86 + rank 1: 0.86 + rank 2: 1.03 + rank 3: 0.96 + rank 4: 1.11 + rank 5: 0.86 + rank 6: 0.91 + rank 7: 1.05 + rank 8: 0.96 + rank 9: 0.91 + rank 10: 1.08 + rank 11: 0.87 + rank 12: 0.91 + rank 13: 1.23 + rank 14: 0.86 + rank 15: 1.23 + rank 16: 1.10 + rank 17: 0.86 + rank 18: 0.89 + rank 19: 0.86 + rank 20: 0.85 + rank 21: 1.06 + rank 22: 0.91 + rank 23: 1.09 + rank 24: 1.24 + rank 25: 1.35 + rank 26: 1.28 + rank 27: 1.62 + rank 28: 1.68 + rank 29: 1.41 + rank 30: 1.55 + rank 31: 1.29 + rank 32: 1.20 + rank 33: 1.34 + rank 34: 1.48 + rank 35: 1.36 + rank 36: 1.18 + rank 37: 1.31 + rank 38: 1.36 + rank 39: 1.18 + rank 40: 1.11 + rank 41: 0.91 + rank 42: 0.84 + rank 43: 1.02 + rank 44: 1.14 + rank 45: 0.87 + rank 46: 1.87 + rank 47: 0.84 + rank 48: 1.23 + rank 49: 1.26 + rank 50: 1.06 + rank 51: 0.99 + rank 52: 1.12 + rank 53: 1.07 + rank 54: 0.90 + rank 55: 1.08 + rank 56: 1.21 + rank 57: 0.86 + rank 58: 1.15 + rank 59: 1.30 + rank 60: 0.82 + rank 61: 0.89 + rank 62: 1.45 + rank 63: 1.27 + params-all-gather: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.02 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.02 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.02 + rank 20: 0.02 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.02 + rank 24: 0.04 + rank 25: 0.03 + rank 26: 0.03 + rank 27: 0.03 + rank 28: 0.03 + rank 29: 0.03 + rank 30: 0.03 + rank 31: 0.03 + rank 32: 0.02 + rank 33: 0.03 + rank 34: 0.02 + rank 35: 0.03 + rank 36: 0.02 + rank 37: 0.02 + rank 38: 0.03 + rank 39: 0.03 + rank 40: 0.02 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.02 + rank 45: 0.02 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.02 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.02 + optimizer-copy-to-main-grad: + rank 0: 2.36 + rank 1: 1.87 + rank 2: 1.80 + rank 3: 2.16 + rank 4: 1.69 + rank 5: 1.94 + rank 6: 1.90 + rank 7: 1.88 + rank 8: 1.92 + rank 9: 2.30 + rank 10: 1.78 + rank 11: 1.85 + rank 12: 1.89 + rank 13: 1.82 + rank 14: 1.98 + rank 15: 2.17 + rank 16: 3.07 + rank 17: 2.57 + rank 18: 2.30 + rank 19: 2.42 + rank 20: 2.62 + rank 21: 2.31 + rank 22: 2.76 + rank 23: 2.41 + rank 24: 2.07 + rank 25: 2.23 + rank 26: 2.16 + rank 27: 2.23 + rank 28: 2.33 + rank 29: 2.43 + rank 30: 2.22 + rank 31: 2.23 + rank 32: 2.10 + rank 33: 1.97 + rank 34: 2.02 + rank 35: 2.55 + rank 36: 2.05 + rank 37: 2.09 + rank 38: 2.55 + rank 39: 1.94 + rank 40: 1.77 + rank 41: 2.08 + rank 42: 1.92 + rank 43: 1.95 + rank 44: 1.90 + rank 45: 2.13 + rank 46: 1.77 + rank 47: 1.77 + rank 48: 1.95 + rank 49: 1.85 + rank 50: 2.50 + rank 51: 1.83 + rank 52: 2.06 + rank 53: 1.83 + rank 54: 1.82 + rank 55: 1.88 + rank 56: 1.80 + rank 57: 2.46 + rank 58: 1.81 + rank 59: 1.82 + rank 60: 1.75 + rank 61: 1.78 + rank 62: 1.86 + rank 63: 1.75 + optimizer-inner-step: + rank 0: 44.80 + rank 1: 45.46 + rank 2: 44.52 + rank 3: 44.59 + rank 4: 44.29 + rank 5: 44.51 + rank 6: 44.83 + rank 7: 45.66 + rank 8: 45.56 + rank 9: 45.55 + rank 10: 45.36 + rank 11: 45.56 + rank 12: 45.58 + rank 13: 45.72 + rank 14: 45.47 + rank 15: 45.29 + rank 16: 45.85 + rank 17: 45.48 + rank 18: 45.54 + rank 19: 45.51 + rank 20: 45.45 + rank 21: 45.41 + rank 22: 46.00 + rank 23: 45.68 + rank 24: 45.23 + rank 25: 45.42 + rank 26: 45.36 + rank 27: 45.46 + rank 28: 46.68 + rank 29: 45.21 + rank 30: 45.42 + rank 31: 45.56 + rank 32: 45.60 + rank 33: 45.35 + rank 34: 45.53 + rank 35: 55.79 + rank 36: 47.83 + rank 37: 48.06 + rank 38: 48.35 + rank 39: 45.39 + rank 40: 45.47 + rank 41: 45.62 + rank 42: 45.61 + rank 43: 45.69 + rank 44: 45.29 + rank 45: 45.86 + rank 46: 45.52 + rank 47: 45.27 + rank 48: 45.48 + rank 49: 45.34 + rank 50: 55.35 + rank 51: 45.22 + rank 52: 45.51 + rank 53: 45.42 + rank 54: 45.61 + rank 55: 45.54 + rank 56: 45.20 + rank 57: 47.57 + rank 58: 45.49 + rank 59: 45.29 + rank 60: 45.58 + rank 61: 45.27 + rank 62: 45.39 + rank 63: 45.33 + optimizer-copy-main-to-model-params: + rank 0: 4.58 + rank 1: 3.96 + rank 2: 3.82 + rank 3: 4.47 + rank 4: 3.73 + rank 5: 4.04 + rank 6: 4.00 + rank 7: 4.00 + rank 8: 3.96 + rank 9: 4.49 + rank 10: 3.81 + rank 11: 3.86 + rank 12: 3.96 + rank 13: 3.86 + rank 14: 4.05 + rank 15: 4.29 + rank 16: 4.52 + rank 17: 4.02 + rank 18: 4.14 + rank 19: 4.08 + rank 20: 4.08 + rank 21: 4.23 + rank 22: 4.69 + rank 23: 4.24 + rank 24: 3.91 + rank 25: 4.21 + rank 26: 4.01 + rank 27: 4.07 + rank 28: 4.37 + rank 29: 4.39 + rank 30: 4.33 + rank 31: 4.02 + rank 32: 4.36 + rank 33: 4.04 + rank 34: 4.14 + rank 35: 4.77 + rank 36: 4.44 + rank 37: 4.41 + rank 38: 4.59 + rank 39: 4.12 + rank 40: 3.82 + rank 41: 4.00 + rank 42: 4.04 + rank 43: 4.06 + rank 44: 4.07 + rank 45: 4.31 + rank 46: 3.79 + rank 47: 3.77 + rank 48: 4.07 + rank 49: 3.94 + rank 50: 4.99 + rank 51: 4.08 + rank 52: 4.38 + rank 53: 3.93 + rank 54: 3.94 + rank 55: 4.00 + rank 56: 3.79 + rank 57: 4.70 + rank 58: 3.83 + rank 59: 3.71 + rank 60: 3.81 + rank 61: 3.73 + rank 62: 3.92 + rank 63: 3.74 + optimizer: + rank 0: 59.58 + rank 1: 58.55 + rank 2: 58.09 + rank 3: 59.18 + rank 4: 57.92 + rank 5: 58.53 + rank 6: 58.88 + rank 7: 58.80 + rank 8: 58.58 + rank 9: 59.30 + rank 10: 58.21 + rank 11: 58.43 + rank 12: 58.59 + rank 13: 58.59 + rank 14: 58.64 + rank 15: 58.78 + rank 16: 59.88 + rank 17: 58.90 + rank 18: 59.10 + rank 19: 58.94 + rank 20: 58.89 + rank 21: 59.08 + rank 22: 60.29 + rank 23: 59.35 + rank 24: 58.23 + rank 25: 58.89 + rank 26: 58.57 + rank 27: 58.76 + rank 28: 60.36 + rank 29: 58.96 + rank 30: 58.99 + rank 31: 58.70 + rank 32: 59.03 + rank 33: 58.39 + rank 34: 58.68 + rank 35: 69.78 + rank 36: 61.35 + rank 37: 61.54 + rank 38: 62.14 + rank 39: 58.49 + rank 40: 58.30 + rank 41: 58.68 + rank 42: 58.73 + rank 43: 58.85 + rank 44: 58.43 + rank 45: 59.37 + rank 46: 58.29 + rank 47: 58.00 + rank 48: 58.68 + rank 49: 58.33 + rank 50: 69.74 + rank 51: 58.32 + rank 52: 59.04 + rank 53: 58.35 + rank 54: 58.54 + rank 55: 58.52 + rank 56: 57.99 + rank 57: 61.69 + rank 58: 58.36 + rank 59: 57.99 + rank 60: 58.39 + rank 61: 57.99 + rank 62: 58.38 + rank 63: 58.07 + interval-time: + rank 0: 0.85 + rank 1: 0.85 + rank 2: 0.89 + rank 3: 0.78 + rank 4: 0.82 + rank 5: 0.86 + rank 6: 0.73 + rank 7: 0.73 + rank 8: 0.77 + rank 9: 0.88 + rank 10: 0.86 + rank 11: 0.85 + rank 12: 0.86 + rank 13: 0.74 + rank 14: 0.86 + rank 15: 0.84 + rank 16: 0.84 + rank 17: 0.84 + rank 18: 0.76 + rank 19: 0.86 + rank 20: 0.74 + rank 21: 0.81 + rank 22: 0.76 + rank 23: 0.83 + rank 24: 1.04 + rank 25: 1.04 + rank 26: 0.99 + rank 27: 0.96 + rank 28: 1.28 + rank 29: 1.29 + rank 30: 1.11 + rank 31: 1.30 + rank 32: 0.83 + rank 33: 0.86 + rank 34: 0.94 + rank 35: 0.95 + rank 36: 0.92 + rank 37: 1.03 + rank 38: 0.92 + rank 39: 0.91 + rank 40: 0.85 + rank 41: 0.76 + rank 42: 0.87 + rank 43: 0.76 + rank 44: 0.74 + rank 45: 0.84 + rank 46: 0.85 + rank 47: 0.82 + rank 48: 0.83 + rank 49: 0.97 + rank 50: 0.85 + rank 51: 0.91 + rank 52: 0.80 + rank 53: 0.85 + rank 54: 0.80 + rank 55: 0.81 + rank 56: 0.86 + rank 57: 0.84 + rank 58: 0.85 + rank 59: 0.85 + rank 60: 0.76 + rank 61: 0.86 + rank 62: 0.75 + rank 63: 2.94 + [2026-10-16 19:41:45] iteration 2002/ 2384 | consumed samples: 64064 | elapsed time per iteration (ms): 230015.7 | throughput per GPU (TFLOP/s/GPU): 23.9 | MFU 7.65% | FLOPs estimator: workload-aware | logical tokens/s: 72939.3 | valid tokens/s: 72939.3 | physical tokens/s: 73345.7 | loss tokens/s: 47929.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.710937E-01 | lm-global-supervised-tokens: 1.102456E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.114318E-02 | lm-token-buffer-mib: 2.336087E+02 | global_load_balancing_loss: 1.000528E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11152.196 | params rms: 0.060386 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:24:25.982047 | finish at 2026-10-17 20:06:11 +times across ranks (ms): + forward-backward: + rank 0: 224080.45 + rank 1: 224157.84 + rank 2: 224159.25 + rank 3: 224159.67 + rank 4: 224160.47 + rank 5: 224159.30 + rank 6: 224155.17 + rank 7: 224159.91 + rank 8: 224127.09 + rank 9: 224127.94 + rank 10: 224127.78 + rank 11: 224128.36 + rank 12: 224128.30 + rank 13: 224128.98 + rank 14: 224126.70 + rank 15: 224126.91 + rank 16: 224151.66 + rank 17: 224149.52 + rank 18: 224150.91 + rank 19: 224155.88 + rank 20: 224156.45 + rank 21: 224149.28 + rank 22: 224144.02 + rank 23: 224152.06 + rank 24: 224145.89 + rank 25: 224147.75 + rank 26: 224143.78 + rank 27: 224148.73 + rank 28: 224150.31 + rank 29: 224148.69 + rank 30: 224145.05 + rank 31: 224143.23 + rank 32: 224049.47 + rank 33: 224125.25 + rank 34: 224126.64 + rank 35: 224125.86 + rank 36: 224127.94 + rank 37: 224125.64 + rank 38: 224124.44 + rank 39: 224129.22 + rank 40: 224110.44 + rank 41: 224111.12 + rank 42: 224109.33 + rank 43: 224109.36 + rank 44: 224108.70 + rank 45: 224107.33 + rank 46: 224112.56 + rank 47: 224112.38 + rank 48: 224154.28 + rank 49: 224150.98 + rank 50: 224150.14 + rank 51: 224155.81 + rank 52: 224153.83 + rank 53: 224147.59 + rank 54: 224147.98 + rank 55: 224155.98 + rank 56: 224148.89 + rank 57: 224148.58 + rank 58: 224143.48 + rank 59: 224147.44 + rank 60: 224147.30 + rank 61: 224144.91 + rank 62: 224150.08 + rank 63: 224147.98 + forward-compute: + rank 0: 129681.28 + rank 1: 129761.77 + rank 2: 129760.95 + rank 3: 129762.64 + rank 4: 129767.59 + rank 5: 129765.45 + rank 6: 129765.07 + rank 7: 129768.66 + rank 8: 129797.91 + rank 9: 129791.25 + rank 10: 129799.32 + rank 11: 129816.66 + rank 12: 129804.86 + rank 13: 129789.95 + rank 14: 129793.68 + rank 15: 129794.47 + rank 16: 129776.74 + rank 17: 129769.16 + rank 18: 129762.78 + rank 19: 129770.45 + rank 20: 129769.88 + rank 21: 129761.34 + rank 22: 129762.17 + rank 23: 129770.61 + rank 24: 129766.20 + rank 25: 129763.97 + rank 26: 129756.16 + rank 27: 129774.87 + rank 28: 129781.79 + rank 29: 129779.73 + rank 30: 129779.99 + rank 31: 129776.07 + rank 32: 61349.07 + rank 33: 61414.96 + rank 34: 61413.30 + rank 35: 61427.74 + rank 36: 61444.02 + rank 37: 61447.02 + rank 38: 61446.49 + rank 39: 61446.27 + rank 40: 61438.23 + rank 41: 61404.32 + rank 42: 61429.64 + rank 43: 61416.92 + rank 44: 61422.59 + rank 45: 61416.75 + rank 46: 61396.64 + rank 47: 61430.07 + rank 48: 61430.19 + rank 49: 61442.70 + rank 50: 61417.38 + rank 51: 61422.38 + rank 52: 61474.73 + rank 53: 61466.71 + rank 54: 61415.66 + rank 55: 61436.83 + rank 56: 61435.36 + rank 57: 61424.67 + rank 58: 61406.72 + rank 59: 61425.70 + rank 60: 61437.93 + rank 61: 61454.15 + rank 62: 61409.29 + rank 63: 61452.98 + backward-compute: + rank 0: 94349.83 + rank 1: 94341.36 + rank 2: 94345.54 + rank 3: 94343.70 + rank 4: 94341.28 + rank 5: 94339.39 + rank 6: 94336.35 + rank 7: 94342.94 + rank 8: 94273.28 + rank 9: 94282.86 + rank 10: 94275.30 + rank 11: 94259.28 + rank 12: 94269.61 + rank 13: 94284.32 + rank 14: 94279.37 + rank 15: 94281.18 + rank 16: 94327.01 + rank 17: 94325.02 + rank 18: 94336.78 + rank 19: 94330.95 + rank 20: 94333.30 + rank 21: 94334.55 + rank 22: 94325.57 + rank 23: 94334.94 + rank 24: 94324.26 + rank 25: 94326.38 + rank 26: 94325.39 + rank 27: 94309.48 + rank 28: 94302.31 + rank 29: 94293.95 + rank 30: 94308.22 + rank 31: 94307.06 + rank 32: 162662.58 + rank 33: 162671.25 + rank 34: 162674.70 + rank 35: 162658.72 + rank 36: 162646.03 + rank 37: 162638.95 + rank 38: 162640.09 + rank 39: 162646.45 + rank 40: 162638.08 + rank 41: 162671.72 + rank 42: 162647.38 + rank 43: 162658.55 + rank 44: 162651.70 + rank 45: 162656.48 + rank 46: 162680.44 + rank 47: 162648.48 + rank 48: 162690.00 + rank 49: 162672.86 + rank 50: 162696.98 + rank 51: 162697.75 + rank 52: 162645.19 + rank 53: 162638.41 + rank 54: 162696.59 + rank 55: 162685.39 + rank 56: 162681.14 + rank 57: 162688.73 + rank 58: 162700.44 + rank 59: 162686.03 + rank 60: 162674.28 + rank 61: 162654.47 + rank 62: 162706.25 + rank 63: 162660.88 + batch-generator: + rank 0: 35.70 + rank 1: 36.20 + rank 2: 36.14 + rank 3: 37.40 + rank 4: 36.83 + rank 5: 38.94 + rank 6: 37.84 + rank 7: 33.17 + rank 8: 61.64 + rank 9: 62.18 + rank 10: 61.93 + rank 11: 67.85 + rank 12: 68.59 + rank 13: 60.16 + rank 14: 63.96 + rank 15: 58.89 + rank 16: 32.64 + rank 17: 35.53 + rank 18: 35.54 + rank 19: 36.60 + rank 20: 35.81 + rank 21: 35.43 + rank 22: 36.27 + rank 23: 31.13 + rank 24: 48.34 + rank 25: 49.36 + rank 26: 48.98 + rank 27: 48.40 + rank 28: 47.04 + rank 29: 51.17 + rank 30: 47.50 + rank 31: 44.62 + rank 32: 43.98 + rank 33: 48.88 + rank 34: 49.09 + rank 35: 52.38 + rank 36: 48.00 + rank 37: 50.69 + rank 38: 42.20 + rank 39: 38.63 + rank 40: 32.90 + rank 41: 36.63 + rank 42: 36.24 + rank 43: 36.74 + rank 44: 35.45 + rank 45: 34.21 + rank 46: 36.02 + rank 47: 29.99 + rank 48: 37.15 + rank 49: 40.08 + rank 50: 45.60 + rank 51: 44.04 + rank 52: 36.88 + rank 53: 48.17 + rank 54: 38.59 + rank 55: 36.42 + rank 56: 31.59 + rank 57: 38.66 + rank 58: 40.57 + rank 59: 36.88 + rank 60: 36.70 + rank 61: 37.57 + rank 62: 37.06 + rank 63: 30.44 + embedding-grads-all-reduce: + rank 0: 0.03 + rank 1: 0.03 + rank 2: 0.03 + rank 3: 0.03 + rank 4: 0.03 + rank 5: 0.03 + rank 6: 0.03 + rank 7: 0.03 + rank 8: 0.03 + rank 9: 0.03 + rank 10: 0.03 + rank 11: 0.03 + rank 12: 0.03 + rank 13: 0.03 + rank 14: 0.03 + rank 15: 0.03 + rank 16: 0.03 + rank 17: 0.03 + rank 18: 0.03 + rank 19: 0.02 + rank 20: 0.03 + rank 21: 0.03 + rank 22: 0.03 + rank 23: 0.03 + rank 24: 0.04 + rank 25: 0.03 + rank 26: 0.04 + rank 27: 0.04 + rank 28: 0.04 + rank 29: 0.04 + rank 30: 0.04 + rank 31: 0.04 + rank 32: 0.03 + rank 33: 0.03 + rank 34: 0.03 + rank 35: 0.03 + rank 36: 0.03 + rank 37: 0.04 + rank 38: 0.03 + rank 39: 0.04 + rank 40: 0.03 + rank 41: 0.03 + rank 42: 0.03 + rank 43: 0.03 + rank 44: 0.03 + rank 45: 0.03 + rank 46: 0.03 + rank 47: 0.03 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.03 + rank 57: 0.03 + rank 58: 0.03 + rank 59: 0.02 + rank 60: 0.03 + rank 61: 0.03 + rank 62: 0.03 + rank 63: 0.03 + all-grads-sync: + rank 0: 0.82 + rank 1: 1.11 + rank 2: 1.03 + rank 3: 0.89 + rank 4: 1.10 + rank 5: 1.10 + rank 6: 0.86 + rank 7: 1.02 + rank 8: 0.89 + rank 9: 1.09 + rank 10: 1.03 + rank 11: 0.83 + rank 12: 0.85 + rank 13: 0.82 + rank 14: 1.07 + rank 15: 1.17 + rank 16: 0.81 + rank 17: 0.94 + rank 18: 1.14 + rank 19: 0.86 + rank 20: 1.21 + rank 21: 0.79 + rank 22: 1.19 + rank 23: 0.87 + rank 24: 1.56 + rank 25: 1.48 + rank 26: 1.36 + rank 27: 1.27 + rank 28: 1.19 + rank 29: 1.11 + rank 30: 1.17 + rank 31: 1.26 + rank 32: 1.27 + rank 33: 1.56 + rank 34: 1.68 + rank 35: 1.50 + rank 36: 1.37 + rank 37: 1.33 + rank 38: 1.52 + rank 39: 1.35 + rank 40: 0.85 + rank 41: 0.89 + rank 42: 0.85 + rank 43: 0.93 + rank 44: 1.12 + rank 45: 0.93 + rank 46: 0.94 + rank 47: 1.07 + rank 48: 0.95 + rank 49: 1.15 + rank 50: 1.09 + rank 51: 1.21 + rank 52: 1.43 + rank 53: 1.32 + rank 54: 0.90 + rank 55: 0.85 + rank 56: 0.94 + rank 57: 0.83 + rank 58: 1.14 + rank 59: 1.35 + rank 60: 1.08 + rank 61: 1.12 + rank 62: 1.00 + rank 63: 0.85 + params-all-gather: + rank 0: 0.02 + rank 1: 0.03 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.02 + rank 6: 0.03 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.03 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.02 + rank 20: 0.02 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.02 + rank 24: 0.04 + rank 25: 0.03 + rank 26: 0.03 + rank 27: 0.03 + rank 28: 0.03 + rank 29: 0.03 + rank 30: 0.04 + rank 31: 0.03 + rank 32: 0.02 + rank 33: 0.02 + rank 34: 0.03 + rank 35: 0.02 + rank 36: 0.03 + rank 37: 0.02 + rank 38: 0.02 + rank 39: 0.02 + rank 40: 0.02 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.02 + rank 45: 0.02 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.02 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.03 + rank 63: 0.02 + optimizer-copy-to-main-grad: + rank 0: 2.22 + rank 1: 1.82 + rank 2: 1.77 + rank 3: 2.17 + rank 4: 1.76 + rank 5: 1.99 + rank 6: 1.98 + rank 7: 1.92 + rank 8: 1.82 + rank 9: 2.16 + rank 10: 1.77 + rank 11: 1.78 + rank 12: 1.86 + rank 13: 1.77 + rank 14: 1.91 + rank 15: 2.15 + rank 16: 2.47 + rank 17: 2.22 + rank 18: 2.25 + rank 19: 2.22 + rank 20: 2.23 + rank 21: 2.27 + rank 22: 2.62 + rank 23: 2.34 + rank 24: 1.95 + rank 25: 2.02 + rank 26: 1.88 + rank 27: 2.23 + rank 28: 2.29 + rank 29: 2.51 + rank 30: 2.09 + rank 31: 2.22 + rank 32: 2.17 + rank 33: 1.92 + rank 34: 2.05 + rank 35: 2.34 + rank 36: 2.10 + rank 37: 2.04 + rank 38: 2.26 + rank 39: 1.94 + rank 40: 1.75 + rank 41: 1.90 + rank 42: 1.86 + rank 43: 1.94 + rank 44: 1.85 + rank 45: 2.11 + rank 46: 1.76 + rank 47: 1.70 + rank 48: 1.97 + rank 49: 1.85 + rank 50: 2.55 + rank 51: 1.82 + rank 52: 2.00 + rank 53: 1.94 + rank 54: 1.81 + rank 55: 1.80 + rank 56: 1.80 + rank 57: 2.40 + rank 58: 1.86 + rank 59: 1.73 + rank 60: 1.78 + rank 61: 1.70 + rank 62: 1.84 + rank 63: 1.82 + optimizer-inner-step: + rank 0: 46.02 + rank 1: 45.91 + rank 2: 45.68 + rank 3: 45.86 + rank 4: 45.61 + rank 5: 45.80 + rank 6: 46.13 + rank 7: 46.09 + rank 8: 38.44 + rank 9: 45.85 + rank 10: 45.72 + rank 11: 45.86 + rank 12: 45.83 + rank 13: 46.04 + rank 14: 45.81 + rank 15: 45.69 + rank 16: 46.22 + rank 17: 45.86 + rank 18: 45.99 + rank 19: 45.91 + rank 20: 45.84 + rank 21: 45.74 + rank 22: 46.39 + rank 23: 45.98 + rank 24: 45.66 + rank 25: 45.93 + rank 26: 45.84 + rank 27: 46.11 + rank 28: 46.33 + rank 29: 45.61 + rank 30: 45.78 + rank 31: 45.99 + rank 32: 45.86 + rank 33: 45.74 + rank 34: 45.80 + rank 35: 48.89 + rank 36: 45.99 + rank 37: 45.83 + rank 38: 45.74 + rank 39: 45.71 + rank 40: 45.71 + rank 41: 45.83 + rank 42: 45.93 + rank 43: 45.99 + rank 44: 45.58 + rank 45: 46.27 + rank 46: 45.85 + rank 47: 45.51 + rank 48: 45.80 + rank 49: 45.72 + rank 50: 57.81 + rank 51: 45.58 + rank 52: 45.91 + rank 53: 45.69 + rank 54: 45.97 + rank 55: 45.86 + rank 56: 45.56 + rank 57: 46.85 + rank 58: 45.85 + rank 59: 45.63 + rank 60: 45.92 + rank 61: 45.61 + rank 62: 45.71 + rank 63: 45.64 + optimizer-copy-main-to-model-params: + rank 0: 4.52 + rank 1: 3.89 + rank 2: 3.81 + rank 3: 4.45 + rank 4: 3.76 + rank 5: 4.11 + rank 6: 4.11 + rank 7: 4.06 + rank 8: 3.88 + rank 9: 4.44 + rank 10: 3.84 + rank 11: 3.83 + rank 12: 3.95 + rank 13: 3.83 + rank 14: 3.99 + rank 15: 4.25 + rank 16: 4.45 + rank 17: 3.94 + rank 18: 4.05 + rank 19: 4.03 + rank 20: 4.05 + rank 21: 4.21 + rank 22: 4.64 + rank 23: 4.21 + rank 24: 3.89 + rank 25: 4.22 + rank 26: 3.97 + rank 27: 4.56 + rank 28: 4.40 + rank 29: 4.41 + rank 30: 4.33 + rank 31: 4.08 + rank 32: 4.18 + rank 33: 3.89 + rank 34: 4.01 + rank 35: 4.53 + rank 36: 4.24 + rank 37: 4.11 + rank 38: 4.48 + rank 39: 3.88 + rank 40: 3.78 + rank 41: 4.05 + rank 42: 3.93 + rank 43: 3.99 + rank 44: 3.95 + rank 45: 4.22 + rank 46: 3.72 + rank 47: 3.67 + rank 48: 4.07 + rank 49: 3.90 + rank 50: 4.95 + rank 51: 4.01 + rank 52: 4.20 + rank 53: 3.92 + rank 54: 3.93 + rank 55: 4.00 + rank 56: 3.72 + rank 57: 4.73 + rank 58: 4.01 + rank 59: 3.69 + rank 60: 3.77 + rank 61: 3.71 + rank 62: 3.89 + rank 63: 3.75 + optimizer: + rank 0: 60.01 + rank 1: 59.04 + rank 2: 58.65 + rank 3: 59.73 + rank 4: 58.53 + rank 5: 59.15 + rank 6: 59.54 + rank 7: 59.44 + rank 8: 58.97 + rank 9: 59.72 + rank 10: 58.79 + rank 11: 58.89 + rank 12: 59.02 + rank 13: 59.07 + rank 14: 59.05 + rank 15: 59.30 + rank 16: 60.30 + rank 17: 59.29 + rank 18: 59.51 + rank 19: 59.40 + rank 20: 59.38 + rank 21: 59.53 + rank 22: 60.72 + rank 23: 59.77 + rank 24: 58.77 + rank 25: 59.44 + rank 26: 59.02 + rank 27: 59.91 + rank 28: 60.10 + rank 29: 59.40 + rank 30: 59.46 + rank 31: 59.31 + rank 32: 59.29 + rank 33: 58.78 + rank 34: 58.99 + rank 35: 62.81 + rank 36: 59.50 + rank 37: 59.17 + rank 38: 59.59 + rank 39: 58.76 + rank 40: 58.73 + rank 41: 59.21 + rank 42: 59.12 + rank 43: 59.26 + rank 44: 58.82 + rank 45: 59.89 + rank 46: 58.75 + rank 47: 58.37 + rank 48: 59.17 + rank 49: 58.79 + rank 50: 72.38 + rank 51: 58.78 + rank 52: 59.41 + rank 53: 58.80 + rank 54: 59.07 + rank 55: 59.02 + rank 56: 58.45 + rank 57: 61.16 + rank 58: 59.10 + rank 59: 58.45 + rank 60: 58.85 + rank 61: 58.47 + rank 62: 58.83 + rank 63: 58.55 + interval-time: + rank 0: 0.85 + rank 1: 0.76 + rank 2: 0.87 + rank 3: 0.78 + rank 4: 0.85 + rank 5: 0.92 + rank 6: 0.76 + rank 7: 0.76 + rank 8: 0.75 + rank 9: 0.87 + rank 10: 0.87 + rank 11: 0.83 + rank 12: 0.85 + rank 13: 0.74 + rank 14: 0.85 + rank 15: 0.84 + rank 16: 0.84 + rank 17: 0.73 + rank 18: 0.75 + rank 19: 0.85 + rank 20: 0.73 + rank 21: 0.82 + rank 22: 0.74 + rank 23: 0.81 + rank 24: 1.35 + rank 25: 1.24 + rank 26: 1.27 + rank 27: 1.44 + rank 28: 1.30 + rank 29: 1.32 + rank 30: 1.27 + rank 31: 1.40 + rank 32: 0.89 + rank 33: 0.85 + rank 34: 0.93 + rank 35: 0.97 + rank 36: 0.94 + rank 37: 0.97 + rank 38: 0.90 + rank 39: 0.91 + rank 40: 0.94 + rank 41: 0.76 + rank 42: 0.83 + rank 43: 0.76 + rank 44: 0.72 + rank 45: 0.81 + rank 46: 0.84 + rank 47: 0.81 + rank 48: 1.08 + rank 49: 1.26 + rank 50: 1.16 + rank 51: 1.28 + rank 52: 1.02 + rank 53: 1.28 + rank 54: 1.03 + rank 55: 1.07 + rank 56: 0.82 + rank 57: 0.81 + rank 58: 0.83 + rank 59: 0.83 + rank 60: 0.75 + rank 61: 0.81 + rank 62: 0.74 + rank 63: 2.84 + [2026-10-16 19:45:30] iteration 2003/ 2384 | consumed samples: 64096 | elapsed time per iteration (ms): 224538.6 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.57% | FLOPs estimator: workload-aware | logical tokens/s: 74718.5 | valid tokens/s: 74718.5 | physical tokens/s: 75799.5 | loss tokens/s: 53656.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.497469E-01 | lm-global-supervised-tokens: 1.204804E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.454252E-02 | lm-token-buffer-mib: 2.383497E+02 | global_load_balancing_loss: 1.001234E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11152.164 | params rms: 0.060385 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:45:49.209222 | finish at 2026-10-17 19:31:19 +times across ranks (ms): + forward-backward: + rank 0: 224311.70 + rank 1: 224314.39 + rank 2: 224313.58 + rank 3: 224313.55 + rank 4: 224313.28 + rank 5: 224315.69 + rank 6: 224313.08 + rank 7: 224312.27 + rank 8: 224302.06 + rank 9: 224306.06 + rank 10: 224307.05 + rank 11: 224302.30 + rank 12: 224306.72 + rank 13: 224307.67 + rank 14: 224309.16 + rank 15: 224305.56 + rank 16: 224298.48 + rank 17: 224301.00 + rank 18: 224300.52 + rank 19: 224302.44 + rank 20: 224301.75 + rank 21: 224299.89 + rank 22: 224297.08 + rank 23: 224300.59 + rank 24: 224303.44 + rank 25: 224304.94 + rank 26: 224304.11 + rank 27: 224304.17 + rank 28: 224307.55 + rank 29: 224306.33 + rank 30: 224300.86 + rank 31: 224301.81 + rank 32: 224285.91 + rank 33: 224287.61 + rank 34: 224286.31 + rank 35: 224288.20 + rank 36: 224287.66 + rank 37: 224288.27 + rank 38: 224286.80 + rank 39: 224287.64 + rank 40: 224290.98 + rank 41: 224293.22 + rank 42: 224289.55 + rank 43: 224286.33 + rank 44: 224287.41 + rank 45: 224288.52 + rank 46: 224293.69 + rank 47: 224295.69 + rank 48: 224303.44 + rank 49: 224303.62 + rank 50: 224302.31 + rank 51: 224304.12 + rank 52: 224301.28 + rank 53: 224301.47 + rank 54: 224303.39 + rank 55: 224306.73 + rank 56: 224300.62 + rank 57: 224301.06 + rank 58: 224298.28 + rank 59: 224299.41 + rank 60: 224299.70 + rank 61: 224298.88 + rank 62: 224302.06 + rank 63: 224300.53 + forward-compute: + rank 0: 117485.52 + rank 1: 117492.61 + rank 2: 117476.53 + rank 3: 117465.77 + rank 4: 117471.25 + rank 5: 117482.45 + rank 6: 117482.30 + rank 7: 117476.14 + rank 8: 117484.55 + rank 9: 117498.55 + rank 10: 117501.58 + rank 11: 117481.43 + rank 12: 117502.92 + rank 13: 117511.59 + rank 14: 117505.61 + rank 15: 117501.55 + rank 16: 117470.88 + rank 17: 117467.89 + rank 18: 117459.20 + rank 19: 117461.40 + rank 20: 117464.62 + rank 21: 117461.88 + rank 22: 117458.92 + rank 23: 117469.91 + rank 24: 117486.45 + rank 25: 117487.55 + rank 26: 117487.09 + rank 27: 117491.41 + rank 28: 117491.88 + rank 29: 117477.67 + rank 30: 117468.44 + rank 31: 117467.93 + rank 32: 62841.57 + rank 33: 62847.09 + rank 34: 62851.76 + rank 35: 62865.70 + rank 36: 62881.33 + rank 37: 62905.95 + rank 38: 62862.82 + rank 39: 62857.16 + rank 40: 62895.06 + rank 41: 62882.95 + rank 42: 62883.61 + rank 43: 62884.24 + rank 44: 62880.43 + rank 45: 62891.86 + rank 46: 62876.87 + rank 47: 62913.00 + rank 48: 62899.50 + rank 49: 62879.84 + rank 50: 62872.63 + rank 51: 62878.71 + rank 52: 62918.49 + rank 53: 62916.64 + rank 54: 62854.18 + rank 55: 62884.06 + rank 56: 62890.86 + rank 57: 62878.69 + rank 58: 62871.26 + rank 59: 62870.90 + rank 60: 62866.13 + rank 61: 62913.18 + rank 62: 62865.82 + rank 63: 62921.67 + backward-compute: + rank 0: 106780.95 + rank 1: 106770.57 + rank 2: 106785.53 + rank 3: 106795.16 + rank 4: 106789.98 + rank 5: 106782.16 + rank 6: 106779.50 + rank 7: 106788.91 + rank 8: 106764.53 + rank 9: 106751.72 + rank 10: 106748.15 + rank 11: 106767.71 + rank 12: 106747.90 + rank 13: 106740.13 + rank 14: 106743.88 + rank 15: 106751.94 + rank 16: 106782.28 + rank 17: 106783.99 + rank 18: 106792.27 + rank 19: 106790.73 + rank 20: 106788.23 + rank 21: 106789.20 + rank 22: 106792.44 + rank 23: 106785.34 + rank 24: 106761.18 + rank 25: 106759.72 + rank 26: 106754.16 + rank 27: 106756.72 + rank 28: 106749.92 + rank 29: 106757.25 + rank 30: 106783.96 + rank 31: 106771.80 + rank 32: 161408.17 + rank 33: 161400.58 + rank 34: 161397.36 + rank 35: 161381.80 + rank 36: 161367.72 + rank 37: 161343.77 + rank 38: 161386.67 + rank 39: 161394.16 + rank 40: 161360.11 + rank 41: 161371.22 + rank 42: 161371.81 + rank 43: 161368.61 + rank 44: 161371.67 + rank 45: 161362.89 + rank 46: 161378.61 + rank 47: 161347.02 + rank 48: 161368.70 + rank 49: 161389.55 + rank 50: 161393.39 + rank 51: 161390.19 + rank 52: 161349.73 + rank 53: 161346.27 + rank 54: 161413.41 + rank 55: 161387.86 + rank 56: 161377.88 + rank 57: 161386.94 + rank 58: 161391.19 + rank 59: 161393.86 + rank 60: 161400.03 + rank 61: 161351.20 + rank 62: 161401.12 + rank 63: 161344.62 + batch-generator: + rank 0: 33.24 + rank 1: 35.05 + rank 2: 35.79 + rank 3: 36.79 + rank 4: 38.38 + rank 5: 38.41 + rank 6: 36.75 + rank 7: 33.49 + rank 8: 41.57 + rank 9: 48.80 + rank 10: 44.16 + rank 11: 43.78 + rank 12: 45.57 + rank 13: 42.32 + rank 14: 49.00 + rank 15: 47.11 + rank 16: 33.29 + rank 17: 35.60 + rank 18: 34.07 + rank 19: 34.28 + rank 20: 35.28 + rank 21: 36.72 + rank 22: 33.19 + rank 23: 30.95 + rank 24: 47.78 + rank 25: 47.33 + rank 26: 51.30 + rank 27: 50.02 + rank 28: 50.07 + rank 29: 52.26 + rank 30: 44.36 + rank 31: 46.10 + rank 32: 48.30 + rank 33: 51.44 + rank 34: 48.50 + rank 35: 52.89 + rank 36: 48.63 + rank 37: 52.91 + rank 38: 41.75 + rank 39: 39.53 + rank 40: 34.81 + rank 41: 38.32 + rank 42: 37.08 + rank 43: 38.32 + rank 44: 34.83 + rank 45: 36.62 + rank 46: 34.69 + rank 47: 31.17 + rank 48: 41.38 + rank 49: 40.71 + rank 50: 43.63 + rank 51: 42.66 + rank 52: 38.94 + rank 53: 47.61 + rank 54: 35.87 + rank 55: 36.82 + rank 56: 33.75 + rank 57: 37.52 + rank 58: 36.33 + rank 59: 36.45 + rank 60: 36.87 + rank 61: 37.54 + rank 62: 36.87 + rank 63: 31.61 + embedding-grads-all-reduce: + rank 0: 0.03 + rank 1: 0.03 + rank 2: 0.03 + rank 3: 0.03 + rank 4: 0.03 + rank 5: 0.03 + rank 6: 0.03 + rank 7: 0.03 + rank 8: 0.03 + rank 9: 0.03 + rank 10: 0.03 + rank 11: 0.03 + rank 12: 0.03 + rank 13: 0.03 + rank 14: 0.03 + rank 15: 0.03 + rank 16: 0.03 + rank 17: 0.03 + rank 18: 0.03 + rank 19: 0.03 + rank 20: 0.03 + rank 21: 0.03 + rank 22: 0.03 + rank 23: 0.03 + rank 24: 0.04 + rank 25: 0.04 + rank 26: 0.04 + rank 27: 0.04 + rank 28: 0.04 + rank 29: 0.05 + rank 30: 0.06 + rank 31: 0.04 + rank 32: 0.03 + rank 33: 0.03 + rank 34: 0.03 + rank 35: 0.03 + rank 36: 0.03 + rank 37: 0.04 + rank 38: 0.03 + rank 39: 0.04 + rank 40: 0.03 + rank 41: 0.03 + rank 42: 0.03 + rank 43: 0.03 + rank 44: 0.03 + rank 45: 0.03 + rank 46: 0.03 + rank 47: 0.03 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.03 + rank 57: 0.03 + rank 58: 0.03 + rank 59: 0.03 + rank 60: 0.03 + rank 61: 0.03 + rank 62: 0.03 + rank 63: 0.03 + all-grads-sync: + rank 0: 0.89 + rank 1: 1.05 + rank 2: 0.80 + rank 3: 1.08 + rank 4: 0.93 + rank 5: 1.00 + rank 6: 1.33 + rank 7: 1.17 + rank 8: 1.11 + rank 9: 1.07 + rank 10: 0.92 + rank 11: 1.08 + rank 12: 0.92 + rank 13: 1.15 + rank 14: 1.44 + rank 15: 0.83 + rank 16: 0.90 + rank 17: 0.88 + rank 18: 0.92 + rank 19: 1.10 + rank 20: 0.83 + rank 21: 0.99 + rank 22: 0.91 + rank 23: 1.18 + rank 24: 1.70 + rank 25: 1.31 + rank 26: 1.42 + rank 27: 1.28 + rank 28: 1.25 + rank 29: 1.36 + rank 30: 1.25 + rank 31: 1.36 + rank 32: 1.41 + rank 33: 1.83 + rank 34: 1.01 + rank 35: 1.35 + rank 36: 1.22 + rank 37: 1.64 + rank 38: 1.46 + rank 39: 1.40 + rank 40: 0.83 + rank 41: 1.06 + rank 42: 1.11 + rank 43: 1.19 + rank 44: 0.84 + rank 45: 0.81 + rank 46: 1.09 + rank 47: 1.13 + rank 48: 1.31 + rank 49: 0.91 + rank 50: 0.90 + rank 51: 1.12 + rank 52: 0.83 + rank 53: 1.18 + rank 54: 1.28 + rank 55: 1.43 + rank 56: 0.88 + rank 57: 1.08 + rank 58: 0.93 + rank 59: 0.92 + rank 60: 0.81 + rank 61: 0.91 + rank 62: 1.20 + rank 63: 1.06 + params-all-gather: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.04 + rank 6: 0.03 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.03 + rank 15: 0.02 + rank 16: 0.02 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.02 + rank 20: 0.02 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.02 + rank 24: 0.04 + rank 25: 0.03 + rank 26: 0.03 + rank 27: 0.03 + rank 28: 0.03 + rank 29: 0.03 + rank 30: 0.03 + rank 31: 0.03 + rank 32: 0.02 + rank 33: 0.02 + rank 34: 0.03 + rank 35: 0.03 + rank 36: 0.03 + rank 37: 0.03 + rank 38: 0.03 + rank 39: 0.03 + rank 40: 0.02 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.02 + rank 45: 0.02 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.02 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.02 + optimizer-copy-to-main-grad: + rank 0: 2.25 + rank 1: 1.97 + rank 2: 1.81 + rank 3: 2.19 + rank 4: 1.72 + rank 5: 2.03 + rank 6: 1.99 + rank 7: 1.93 + rank 8: 1.79 + rank 9: 2.27 + rank 10: 1.80 + rank 11: 1.73 + rank 12: 1.95 + rank 13: 1.72 + rank 14: 2.51 + rank 15: 2.03 + rank 16: 2.37 + rank 17: 2.39 + rank 18: 2.25 + rank 19: 2.16 + rank 20: 2.25 + rank 21: 2.24 + rank 22: 2.54 + rank 23: 2.37 + rank 24: 1.86 + rank 25: 2.08 + rank 26: 2.21 + rank 27: 2.13 + rank 28: 2.36 + rank 29: 2.29 + rank 30: 2.15 + rank 31: 2.26 + rank 32: 2.00 + rank 33: 1.98 + rank 34: 1.97 + rank 35: 2.74 + rank 36: 2.11 + rank 37: 2.17 + rank 38: 2.29 + rank 39: 1.90 + rank 40: 1.72 + rank 41: 1.97 + rank 42: 1.97 + rank 43: 1.95 + rank 44: 1.87 + rank 45: 2.04 + rank 46: 1.79 + rank 47: 1.79 + rank 48: 1.96 + rank 49: 1.82 + rank 50: 2.45 + rank 51: 1.88 + rank 52: 2.10 + rank 53: 1.87 + rank 54: 1.82 + rank 55: 1.81 + rank 56: 1.75 + rank 57: 2.54 + rank 58: 1.87 + rank 59: 1.74 + rank 60: 1.81 + rank 61: 1.81 + rank 62: 1.95 + rank 63: 1.72 + optimizer-inner-step: + rank 0: 42.66 + rank 1: 42.49 + rank 2: 42.26 + rank 3: 42.47 + rank 4: 42.25 + rank 5: 42.41 + rank 6: 42.80 + rank 7: 42.74 + rank 8: 42.54 + rank 9: 42.44 + rank 10: 42.39 + rank 11: 42.47 + rank 12: 42.48 + rank 13: 42.67 + rank 14: 43.62 + rank 15: 42.32 + rank 16: 42.91 + rank 17: 42.49 + rank 18: 42.69 + rank 19: 42.49 + rank 20: 42.41 + rank 21: 42.39 + rank 22: 43.04 + rank 23: 42.64 + rank 24: 42.18 + rank 25: 42.44 + rank 26: 42.24 + rank 27: 42.52 + rank 28: 46.44 + rank 29: 42.17 + rank 30: 42.37 + rank 31: 42.43 + rank 32: 42.53 + rank 33: 42.43 + rank 34: 42.54 + rank 35: 51.66 + rank 36: 42.95 + rank 37: 42.55 + rank 38: 42.32 + rank 39: 42.31 + rank 40: 42.48 + rank 41: 42.55 + rank 42: 42.59 + rank 43: 42.69 + rank 44: 42.24 + rank 45: 42.83 + rank 46: 42.48 + rank 47: 42.18 + rank 48: 42.44 + rank 49: 42.44 + rank 50: 54.32 + rank 51: 42.22 + rank 52: 42.53 + rank 53: 42.39 + rank 54: 42.56 + rank 55: 42.49 + rank 56: 42.16 + rank 57: 47.65 + rank 58: 42.44 + rank 59: 42.26 + rank 60: 42.62 + rank 61: 42.27 + rank 62: 42.30 + rank 63: 42.21 + optimizer-copy-main-to-model-params: + rank 0: 4.41 + rank 1: 4.18 + rank 2: 3.76 + rank 3: 4.36 + rank 4: 3.80 + rank 5: 4.42 + rank 6: 4.15 + rank 7: 4.12 + rank 8: 3.85 + rank 9: 4.59 + rank 10: 3.90 + rank 11: 3.78 + rank 12: 3.98 + rank 13: 3.80 + rank 14: 4.96 + rank 15: 4.21 + rank 16: 4.46 + rank 17: 3.98 + rank 18: 4.02 + rank 19: 4.02 + rank 20: 4.08 + rank 21: 4.13 + rank 22: 4.58 + rank 23: 4.19 + rank 24: 3.85 + rank 25: 4.20 + rank 26: 4.01 + rank 27: 4.05 + rank 28: 4.51 + rank 29: 4.36 + rank 30: 4.32 + rank 31: 4.04 + rank 32: 4.27 + rank 33: 4.10 + rank 34: 4.01 + rank 35: 5.27 + rank 36: 4.36 + rank 37: 4.54 + rank 38: 4.59 + rank 39: 3.97 + rank 40: 3.81 + rank 41: 4.18 + rank 42: 3.99 + rank 43: 4.00 + rank 44: 4.02 + rank 45: 4.30 + rank 46: 3.77 + rank 47: 3.76 + rank 48: 4.05 + rank 49: 3.95 + rank 50: 5.03 + rank 51: 3.97 + rank 52: 4.21 + rank 53: 3.95 + rank 54: 3.92 + rank 55: 3.92 + rank 56: 3.78 + rank 57: 4.71 + rank 58: 4.02 + rank 59: 3.79 + rank 60: 3.80 + rank 61: 3.77 + rank 62: 3.95 + rank 63: 3.65 + optimizer: + rank 0: 54.44 + rank 1: 53.87 + rank 2: 53.09 + rank 3: 54.10 + rank 4: 53.11 + rank 5: 54.10 + rank 6: 54.18 + rank 7: 54.13 + rank 8: 53.51 + rank 9: 54.42 + rank 10: 53.42 + rank 11: 53.34 + rank 12: 53.63 + rank 13: 53.57 + rank 14: 56.19 + rank 15: 53.76 + rank 16: 54.89 + rank 17: 53.89 + rank 18: 54.07 + rank 19: 53.90 + rank 20: 53.95 + rank 21: 54.00 + rank 22: 55.20 + rank 23: 54.35 + rank 24: 53.17 + rank 25: 53.97 + rank 26: 53.52 + rank 27: 53.81 + rank 28: 58.27 + rank 29: 53.97 + rank 30: 53.98 + rank 31: 53.82 + rank 32: 54.05 + rank 33: 53.65 + rank 34: 53.67 + rank 35: 64.51 + rank 36: 54.64 + rank 37: 54.37 + rank 38: 54.25 + rank 39: 53.46 + rank 40: 53.38 + rank 41: 53.93 + rank 42: 53.76 + rank 43: 53.84 + rank 44: 53.43 + rank 45: 54.44 + rank 46: 53.32 + rank 47: 53.04 + rank 48: 53.68 + rank 49: 53.46 + rank 50: 66.83 + rank 51: 53.31 + rank 52: 53.95 + rank 53: 53.45 + rank 54: 53.57 + rank 55: 53.48 + rank 56: 53.02 + rank 57: 59.95 + rank 58: 53.61 + rank 59: 53.14 + rank 60: 53.48 + rank 61: 53.09 + rank 62: 53.41 + rank 63: 52.93 + interval-time: + rank 0: 0.83 + rank 1: 0.84 + rank 2: 0.85 + rank 3: 0.75 + rank 4: 0.86 + rank 5: 0.93 + rank 6: 0.78 + rank 7: 0.78 + rank 8: 7.34 + rank 9: 7.37 + rank 10: 7.37 + rank 11: 7.36 + rank 12: 7.35 + rank 13: 7.36 + rank 14: 7.34 + rank 15: 7.33 + rank 16: 8.23 + rank 17: 8.22 + rank 18: 8.22 + rank 19: 8.26 + rank 20: 8.22 + rank 21: 8.22 + rank 22: 8.21 + rank 23: 8.22 + rank 24: 0.95 + rank 25: 0.96 + rank 26: 0.91 + rank 27: 0.96 + rank 28: 0.93 + rank 29: 0.95 + rank 30: 1.01 + rank 31: 0.95 + rank 32: 0.84 + rank 33: 0.87 + rank 34: 0.85 + rank 35: 1.07 + rank 36: 0.92 + rank 37: 0.83 + rank 38: 0.91 + rank 39: 0.79 + rank 40: 0.85 + rank 41: 0.85 + rank 42: 0.84 + rank 43: 0.76 + rank 44: 0.72 + rank 45: 0.83 + rank 46: 0.77 + rank 47: 0.82 + rank 48: 0.93 + rank 49: 0.99 + rank 50: 0.85 + rank 51: 0.95 + rank 52: 0.82 + rank 53: 0.98 + rank 54: 0.91 + rank 55: 0.91 + rank 56: 0.83 + rank 57: 0.83 + rank 58: 0.84 + rank 59: 0.73 + rank 60: 0.73 + rank 61: 0.82 + rank 62: 0.72 + rank 63: 2.88 + [2026-10-16 19:49:12] iteration 2004/ 2384 | consumed samples: 64128 | elapsed time per iteration (ms): 222299.8 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.45% | FLOPs estimator: workload-aware | logical tokens/s: 75471.1 | valid tokens/s: 75471.1 | physical tokens/s: 75695.1 | loss tokens/s: 53406.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.764616E-01 | lm-global-supervised-tokens: 1.187218E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.422352E-02 | lm-token-buffer-mib: 2.328670E+02 | global_load_balancing_loss: 1.000747E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11152.136 | params rms: 0.060385 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:27:53.922791 | finish at 2026-10-17 19:17:06 +times across ranks (ms): + forward-backward: + rank 0: 222070.88 + rank 1: 222066.81 + rank 2: 222071.23 + rank 3: 222067.58 + rank 4: 222067.92 + rank 5: 222071.08 + rank 6: 222068.14 + rank 7: 222072.19 + rank 8: 222059.19 + rank 9: 222055.22 + rank 10: 222059.33 + rank 11: 222059.64 + rank 12: 222058.39 + rank 13: 222061.08 + rank 14: 222058.81 + rank 15: 222058.19 + rank 16: 222050.34 + rank 17: 222050.08 + rank 18: 222051.59 + rank 19: 222053.11 + rank 20: 222053.56 + rank 21: 222053.33 + rank 22: 222051.39 + rank 23: 222051.19 + rank 24: 222057.84 + rank 25: 222062.25 + rank 26: 222058.66 + rank 27: 222062.84 + rank 28: 222061.27 + rank 29: 222065.38 + rank 30: 222056.73 + rank 31: 222057.33 + rank 32: 222054.72 + rank 33: 222050.64 + rank 34: 222054.84 + rank 35: 222051.02 + rank 36: 222051.30 + rank 37: 222054.44 + rank 38: 222051.44 + rank 39: 222055.72 + rank 40: 222048.92 + rank 41: 222044.50 + rank 42: 222046.95 + rank 43: 222046.92 + rank 44: 222044.11 + rank 45: 222048.16 + rank 46: 222050.67 + rank 47: 222050.09 + rank 48: 222056.94 + rank 49: 222055.70 + rank 50: 222056.70 + rank 51: 222058.39 + rank 52: 222055.55 + rank 53: 222056.64 + rank 54: 222058.78 + rank 55: 222059.91 + rank 56: 222058.70 + rank 57: 222059.69 + rank 58: 222057.73 + rank 59: 222058.53 + rank 60: 222056.67 + rank 61: 222056.94 + rank 62: 222061.27 + rank 63: 222060.77 + forward-compute: + rank 0: 121378.35 + rank 1: 121370.63 + rank 2: 121368.64 + rank 3: 121367.38 + rank 4: 121370.44 + rank 5: 121370.80 + rank 6: 121362.93 + rank 7: 121366.48 + rank 8: 121412.57 + rank 9: 121408.31 + rank 10: 121413.87 + rank 11: 121408.91 + rank 12: 121409.49 + rank 13: 121416.54 + rank 14: 121414.30 + rank 15: 121409.64 + rank 16: 121364.23 + rank 17: 121358.64 + rank 18: 121362.34 + rank 19: 121362.73 + rank 20: 121366.90 + rank 21: 121368.16 + rank 22: 121356.02 + rank 23: 121357.38 + rank 24: 121366.16 + rank 25: 121366.70 + rank 26: 121365.13 + rank 27: 121368.92 + rank 28: 121367.68 + rank 29: 121363.58 + rank 30: 121359.02 + rank 31: 121357.34 + rank 32: 61165.70 + rank 33: 61167.77 + rank 34: 61173.12 + rank 35: 61183.14 + rank 36: 61201.87 + rank 37: 61214.48 + rank 38: 61187.21 + rank 39: 61190.99 + rank 40: 61212.42 + rank 41: 61169.65 + rank 42: 61187.74 + rank 43: 61190.68 + rank 44: 61181.39 + rank 45: 61210.62 + rank 46: 61154.30 + rank 47: 61194.95 + rank 48: 61167.38 + rank 49: 61157.47 + rank 50: 61160.91 + rank 51: 61148.94 + rank 52: 61198.15 + rank 53: 61185.60 + rank 54: 61141.40 + rank 55: 61169.89 + rank 56: 61184.43 + rank 57: 61172.94 + rank 58: 61156.80 + rank 59: 61153.41 + rank 60: 61188.04 + rank 61: 61182.12 + rank 62: 61171.75 + rank 63: 61214.13 + backward-compute: + rank 0: 100636.80 + rank 1: 100637.49 + rank 2: 100644.84 + rank 3: 100642.50 + rank 4: 100642.93 + rank 5: 100643.03 + rank 6: 100644.95 + rank 7: 100648.83 + rank 8: 100598.29 + rank 9: 100590.76 + rank 10: 100589.19 + rank 11: 100599.65 + rank 12: 100595.65 + rank 13: 100592.33 + rank 14: 100592.34 + rank 15: 100601.22 + rank 16: 100641.67 + rank 17: 100640.30 + rank 18: 100641.85 + rank 19: 100637.81 + rank 20: 100636.99 + rank 21: 100634.25 + rank 22: 100645.39 + rank 23: 100647.01 + rank 24: 100637.15 + rank 25: 100633.84 + rank 26: 100631.34 + rank 27: 100630.30 + rank 28: 100626.75 + rank 29: 100631.34 + rank 30: 100652.34 + rank 31: 100643.91 + rank 32: 160842.56 + rank 33: 160834.66 + rank 34: 160835.25 + rank 35: 160818.58 + rank 36: 160804.14 + rank 37: 160792.59 + rank 38: 160817.28 + rank 39: 160819.39 + rank 40: 160799.14 + rank 41: 160838.97 + rank 42: 160825.12 + rank 43: 160821.89 + rank 44: 160829.55 + rank 45: 160802.27 + rank 46: 160859.06 + rank 47: 160820.31 + rank 48: 160854.33 + rank 49: 160862.95 + rank 50: 160857.53 + rank 51: 160872.44 + rank 52: 160825.11 + rank 53: 160833.08 + rank 54: 160878.30 + rank 55: 160854.89 + rank 56: 160842.78 + rank 57: 160851.08 + rank 58: 160864.94 + rank 59: 160868.05 + rank 60: 160833.12 + rank 61: 160840.16 + rank 62: 160854.33 + rank 63: 160810.48 + batch-generator: + rank 0: 33.90 + rank 1: 37.29 + rank 2: 38.25 + rank 3: 38.32 + rank 4: 37.74 + rank 5: 39.10 + rank 6: 38.10 + rank 7: 35.09 + rank 8: 51.23 + rank 9: 52.52 + rank 10: 49.25 + rank 11: 51.09 + rank 12: 51.30 + rank 13: 49.58 + rank 14: 52.74 + rank 15: 47.13 + rank 16: 35.17 + rank 17: 35.99 + rank 18: 35.62 + rank 19: 35.70 + rank 20: 34.82 + rank 21: 36.04 + rank 22: 35.08 + rank 23: 33.01 + rank 24: 46.69 + rank 25: 47.67 + rank 26: 48.33 + rank 27: 49.36 + rank 28: 47.31 + rank 29: 49.19 + rank 30: 44.03 + rank 31: 43.44 + rank 32: 50.85 + rank 33: 51.37 + rank 34: 49.27 + rank 35: 53.89 + rank 36: 50.25 + rank 37: 50.33 + rank 38: 42.66 + rank 39: 43.47 + rank 40: 32.89 + rank 41: 39.92 + rank 42: 40.09 + rank 43: 36.31 + rank 44: 34.98 + rank 45: 34.93 + rank 46: 39.14 + rank 47: 31.99 + rank 48: 43.47 + rank 49: 42.01 + rank 50: 46.39 + rank 51: 43.45 + rank 52: 34.30 + rank 53: 44.10 + rank 54: 38.68 + rank 55: 37.97 + rank 56: 30.80 + rank 57: 37.68 + rank 58: 41.21 + rank 59: 39.61 + rank 60: 37.55 + rank 61: 35.91 + rank 62: 35.95 + rank 63: 30.30 + embedding-grads-all-reduce: + rank 0: 0.03 + rank 1: 0.03 + rank 2: 0.03 + rank 3: 0.03 + rank 4: 0.03 + rank 5: 0.03 + rank 6: 0.03 + rank 7: 0.03 + rank 8: 0.03 + rank 9: 0.03 + rank 10: 0.03 + rank 11: 0.03 + rank 12: 0.03 + rank 13: 0.02 + rank 14: 0.03 + rank 15: 0.03 + rank 16: 0.03 + rank 17: 0.03 + rank 18: 0.03 + rank 19: 0.02 + rank 20: 0.03 + rank 21: 0.02 + rank 22: 0.03 + rank 23: 0.03 + rank 24: 0.04 + rank 25: 0.04 + rank 26: 0.04 + rank 27: 0.04 + rank 28: 0.04 + rank 29: 0.04 + rank 30: 0.03 + rank 31: 0.04 + rank 32: 0.03 + rank 33: 0.03 + rank 34: 0.04 + rank 35: 0.04 + rank 36: 0.03 + rank 37: 0.04 + rank 38: 0.03 + rank 39: 0.04 + rank 40: 0.03 + rank 41: 0.03 + rank 42: 0.03 + rank 43: 0.03 + rank 44: 0.03 + rank 45: 0.03 + rank 46: 0.03 + rank 47: 0.04 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.04 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.03 + rank 57: 0.03 + rank 58: 0.04 + rank 59: 0.02 + rank 60: 0.03 + rank 61: 0.03 + rank 62: 0.03 + rank 63: 0.03 + all-grads-sync: + rank 0: 1.13 + rank 1: 0.91 + rank 2: 1.16 + rank 3: 1.30 + rank 4: 1.08 + rank 5: 1.25 + rank 6: 1.08 + rank 7: 1.21 + rank 8: 1.23 + rank 9: 0.90 + rank 10: 0.82 + rank 11: 1.37 + rank 12: 0.85 + rank 13: 0.80 + rank 14: 1.30 + rank 15: 0.90 + rank 16: 1.20 + rank 17: 0.82 + rank 18: 0.92 + rank 19: 1.00 + rank 20: 0.93 + rank 21: 0.90 + rank 22: 0.88 + rank 23: 0.90 + rank 24: 1.24 + rank 25: 1.21 + rank 26: 1.25 + rank 27: 2.05 + rank 28: 1.90 + rank 29: 1.70 + rank 30: 1.12 + rank 31: 1.33 + rank 32: 1.72 + rank 33: 1.25 + rank 34: 1.67 + rank 35: 1.22 + rank 36: 1.29 + rank 37: 1.26 + rank 38: 1.36 + rank 39: 1.19 + rank 40: 0.91 + rank 41: 1.20 + rank 42: 1.22 + rank 43: 0.79 + rank 44: 1.09 + rank 45: 1.34 + rank 46: 1.31 + rank 47: 1.35 + rank 48: 1.26 + rank 49: 1.11 + rank 50: 1.04 + rank 51: 1.31 + rank 52: 0.83 + rank 53: 1.25 + rank 54: 0.93 + rank 55: 1.37 + rank 56: 0.82 + rank 57: 1.13 + rank 58: 0.90 + rank 59: 0.93 + rank 60: 0.93 + rank 61: 0.86 + rank 62: 0.95 + rank 63: 1.28 + params-all-gather: + rank 0: 0.03 + rank 1: 0.03 + rank 2: 0.02 + rank 3: 0.03 + rank 4: 0.03 + rank 5: 0.03 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.02 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.02 + rank 20: 0.02 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.02 + rank 24: 0.03 + rank 25: 0.03 + rank 26: 0.04 + rank 27: 0.03 + rank 28: 0.03 + rank 29: 0.03 + rank 30: 0.03 + rank 31: 0.03 + rank 32: 0.02 + rank 33: 0.02 + rank 34: 0.03 + rank 35: 0.02 + rank 36: 0.02 + rank 37: 0.02 + rank 38: 0.03 + rank 39: 0.02 + rank 40: 0.02 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.03 + rank 45: 0.02 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.02 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.02 + optimizer-copy-to-main-grad: + rank 0: 2.24 + rank 1: 1.88 + rank 2: 1.79 + rank 3: 2.23 + rank 4: 1.74 + rank 5: 1.95 + rank 6: 1.93 + rank 7: 1.97 + rank 8: 1.88 + rank 9: 2.24 + rank 10: 1.94 + rank 11: 1.85 + rank 12: 1.92 + rank 13: 1.47 + rank 14: 1.99 + rank 15: 2.15 + rank 16: 2.51 + rank 17: 2.25 + rank 18: 2.29 + rank 19: 2.23 + rank 20: 2.29 + rank 21: 2.35 + rank 22: 2.73 + rank 23: 2.46 + rank 24: 2.14 + rank 25: 2.14 + rank 26: 2.13 + rank 27: 2.28 + rank 28: 2.55 + rank 29: 2.54 + rank 30: 2.26 + rank 31: 2.24 + rank 32: 2.04 + rank 33: 1.90 + rank 34: 2.03 + rank 35: 2.22 + rank 36: 2.08 + rank 37: 1.99 + rank 38: 2.32 + rank 39: 1.99 + rank 40: 1.83 + rank 41: 1.92 + rank 42: 1.95 + rank 43: 1.95 + rank 44: 1.91 + rank 45: 2.12 + rank 46: 1.74 + rank 47: 1.75 + rank 48: 1.93 + rank 49: 1.82 + rank 50: 2.50 + rank 51: 1.90 + rank 52: 1.98 + rank 53: 1.87 + rank 54: 1.83 + rank 55: 1.86 + rank 56: 1.86 + rank 57: 2.46 + rank 58: 1.91 + rank 59: 1.79 + rank 60: 1.82 + rank 61: 1.75 + rank 62: 1.96 + rank 63: 1.80 + optimizer-inner-step: + rank 0: 48.69 + rank 1: 48.67 + rank 2: 48.40 + rank 3: 48.58 + rank 4: 48.40 + rank 5: 48.60 + rank 6: 48.92 + rank 7: 48.84 + rank 8: 48.67 + rank 9: 48.59 + rank 10: 48.52 + rank 11: 48.63 + rank 12: 48.65 + rank 13: 48.86 + rank 14: 48.46 + rank 15: 48.45 + rank 16: 48.98 + rank 17: 48.65 + rank 18: 48.77 + rank 19: 48.65 + rank 20: 48.60 + rank 21: 48.49 + rank 22: 49.15 + rank 23: 48.71 + rank 24: 48.33 + rank 25: 48.66 + rank 26: 48.55 + rank 27: 48.70 + rank 28: 48.99 + rank 29: 48.51 + rank 30: 48.57 + rank 31: 48.71 + rank 32: 48.20 + rank 33: 48.16 + rank 34: 48.58 + rank 35: 50.00 + rank 36: 48.73 + rank 37: 48.25 + rank 38: 48.51 + rank 39: 48.42 + rank 40: 48.57 + rank 41: 48.66 + rank 42: 48.64 + rank 43: 48.76 + rank 44: 48.33 + rank 45: 49.01 + rank 46: 48.66 + rank 47: 48.35 + rank 48: 48.61 + rank 49: 48.54 + rank 50: 54.69 + rank 51: 48.34 + rank 52: 48.65 + rank 53: 48.48 + rank 54: 48.73 + rank 55: 55.75 + rank 56: 48.23 + rank 57: 49.45 + rank 58: 48.51 + rank 59: 48.39 + rank 60: 48.73 + rank 61: 48.42 + rank 62: 48.48 + rank 63: 48.33 + optimizer-copy-main-to-model-params: + rank 0: 4.36 + rank 1: 3.91 + rank 2: 3.74 + rank 3: 4.67 + rank 4: 3.74 + rank 5: 4.06 + rank 6: 4.10 + rank 7: 4.07 + rank 8: 3.87 + rank 9: 4.50 + rank 10: 3.92 + rank 11: 3.83 + rank 12: 3.95 + rank 13: 3.78 + rank 14: 4.01 + rank 15: 4.22 + rank 16: 4.42 + rank 17: 3.98 + rank 18: 4.05 + rank 19: 4.01 + rank 20: 4.02 + rank 21: 4.29 + rank 22: 4.70 + rank 23: 4.30 + rank 24: 3.90 + rank 25: 4.25 + rank 26: 3.95 + rank 27: 4.10 + rank 28: 4.38 + rank 29: 4.43 + rank 30: 4.39 + rank 31: 4.03 + rank 32: 4.22 + rank 33: 4.00 + rank 34: 4.02 + rank 35: 4.59 + rank 36: 4.22 + rank 37: 4.15 + rank 38: 4.59 + rank 39: 4.00 + rank 40: 3.79 + rank 41: 4.06 + rank 42: 3.97 + rank 43: 4.03 + rank 44: 3.93 + rank 45: 4.26 + rank 46: 3.77 + rank 47: 3.72 + rank 48: 4.08 + rank 49: 3.88 + rank 50: 5.01 + rank 51: 3.98 + rank 52: 4.20 + rank 53: 3.90 + rank 54: 3.87 + rank 55: 3.95 + rank 56: 3.97 + rank 57: 4.78 + rank 58: 3.94 + rank 59: 3.75 + rank 60: 3.75 + rank 61: 3.76 + rank 62: 3.98 + rank 63: 3.77 + optimizer: + rank 0: 60.73 + rank 1: 59.98 + rank 2: 59.44 + rank 3: 60.87 + rank 4: 59.44 + rank 5: 60.04 + rank 6: 60.43 + rank 7: 60.35 + rank 8: 59.91 + rank 9: 60.69 + rank 10: 59.79 + rank 11: 59.80 + rank 12: 59.98 + rank 13: 59.95 + rank 14: 59.99 + rank 15: 60.16 + rank 16: 61.23 + rank 17: 60.26 + rank 18: 60.42 + rank 19: 60.30 + rank 20: 60.27 + rank 21: 60.52 + rank 22: 61.71 + rank 23: 60.79 + rank 24: 59.61 + rank 25: 60.36 + rank 26: 59.87 + rank 27: 60.19 + rank 28: 60.92 + rank 29: 60.48 + rank 30: 60.44 + rank 31: 60.15 + rank 32: 60.20 + rank 33: 59.78 + rank 34: 59.96 + rank 35: 62.43 + rank 36: 60.37 + rank 37: 60.13 + rank 38: 60.64 + rank 39: 59.77 + rank 40: 59.75 + rank 41: 60.14 + rank 42: 60.03 + rank 43: 60.20 + rank 44: 59.68 + rank 45: 60.76 + rank 46: 59.72 + rank 47: 59.37 + rank 48: 60.08 + rank 49: 59.73 + rank 50: 67.43 + rank 51: 59.66 + rank 52: 60.29 + rank 53: 59.70 + rank 54: 59.91 + rank 55: 67.02 + rank 56: 59.55 + rank 57: 62.00 + rank 58: 59.85 + rank 59: 59.44 + rank 60: 59.78 + rank 61: 59.50 + rank 62: 59.87 + rank 63: 59.44 + interval-time: + rank 0: 0.82 + rank 1: 0.83 + rank 2: 0.84 + rank 3: 0.76 + rank 4: 0.85 + rank 5: 0.87 + rank 6: 0.77 + rank 7: 0.72 + rank 8: 0.74 + rank 9: 0.81 + rank 10: 0.84 + rank 11: 0.83 + rank 12: 0.84 + rank 13: 0.70 + rank 14: 0.84 + rank 15: 0.80 + rank 16: 0.83 + rank 17: 0.82 + rank 18: 0.78 + rank 19: 0.84 + rank 20: 0.73 + rank 21: 0.83 + rank 22: 0.76 + rank 23: 0.83 + rank 24: 1.36 + rank 25: 1.28 + rank 26: 1.35 + rank 27: 1.20 + rank 28: 1.27 + rank 29: 1.38 + rank 30: 1.38 + rank 31: 1.30 + rank 32: 0.86 + rank 33: 0.84 + rank 34: 0.84 + rank 35: 0.96 + rank 36: 0.92 + rank 37: 0.84 + rank 38: 0.90 + rank 39: 0.80 + rank 40: 0.84 + rank 41: 0.85 + rank 42: 0.84 + rank 43: 0.74 + rank 44: 0.73 + rank 45: 0.83 + rank 46: 0.74 + rank 47: 0.79 + rank 48: 0.92 + rank 49: 0.98 + rank 50: 0.90 + rank 51: 0.94 + rank 52: 0.81 + rank 53: 0.97 + rank 54: 0.90 + rank 55: 0.91 + rank 56: 0.81 + rank 57: 0.82 + rank 58: 0.83 + rank 59: 0.73 + rank 60: 0.75 + rank 61: 0.83 + rank 62: 0.74 + rank 63: 2.35 + [2026-10-16 19:52:52] iteration 2005/ 2384 | consumed samples: 64160 | elapsed time per iteration (ms): 220002.9 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.30% | FLOPs estimator: workload-aware | logical tokens/s: 76259.0 | valid tokens/s: 76259.0 | physical tokens/s: 76784.7 | loss tokens/s: 57952.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.056471E-01 | lm-global-supervised-tokens: 1.274966E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.235823E-02 | lm-token-buffer-mib: 2.327853E+02 | global_load_balancing_loss: 9.996746E-01 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11152.110 | params rms: 0.060385 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:09:41.092822 | finish at 2026-10-17 19:02:33 +times across ranks (ms): + forward-backward: + rank 0: 219771.39 + rank 1: 219771.30 + rank 2: 219768.09 + rank 3: 219771.75 + rank 4: 219776.28 + rank 5: 219773.19 + rank 6: 219770.66 + rank 7: 219772.83 + rank 8: 219769.53 + rank 9: 219770.25 + rank 10: 219771.28 + rank 11: 219768.50 + rank 12: 219768.23 + rank 13: 219773.75 + rank 14: 219765.81 + rank 15: 219770.59 + rank 16: 219763.27 + rank 17: 219761.91 + rank 18: 219763.44 + rank 19: 219762.69 + rank 20: 219765.61 + rank 21: 219766.22 + rank 22: 219765.88 + rank 23: 219766.69 + rank 24: 219766.45 + rank 25: 219765.86 + rank 26: 219763.42 + rank 27: 219764.06 + rank 28: 219765.25 + rank 29: 219765.00 + rank 30: 219766.33 + rank 31: 219766.80 + rank 32: 219754.06 + rank 33: 219754.22 + rank 34: 219749.17 + rank 35: 219753.08 + rank 36: 219757.17 + rank 37: 219753.22 + rank 38: 219752.97 + rank 39: 219755.78 + rank 40: 219759.70 + rank 41: 219757.91 + rank 42: 219757.36 + rank 43: 219754.47 + rank 44: 219752.73 + rank 45: 219756.17 + rank 46: 219759.06 + rank 47: 219761.16 + rank 48: 219766.45 + rank 49: 219764.44 + rank 50: 219765.17 + rank 51: 219765.66 + rank 52: 219765.44 + rank 53: 219768.72 + rank 54: 219769.55 + rank 55: 219770.94 + rank 56: 219769.19 + rank 57: 219768.20 + rank 58: 219765.73 + rank 59: 219765.23 + rank 60: 219765.84 + rank 61: 219765.70 + rank 62: 219771.52 + rank 63: 219770.03 + forward-compute: + rank 0: 117276.18 + rank 1: 117275.50 + rank 2: 117241.73 + rank 3: 117234.30 + rank 4: 117235.01 + rank 5: 117243.16 + rank 6: 117250.16 + rank 7: 117258.91 + rank 8: 117193.77 + rank 9: 117203.16 + rank 10: 117194.29 + rank 11: 117194.47 + rank 12: 117205.10 + rank 13: 117211.28 + rank 14: 117203.00 + rank 15: 117211.26 + rank 16: 117229.16 + rank 17: 117223.30 + rank 18: 117222.56 + rank 19: 117220.72 + rank 20: 117217.12 + rank 21: 117219.88 + rank 22: 117223.10 + rank 23: 117222.06 + rank 24: 117142.95 + rank 25: 117136.58 + rank 26: 117133.34 + rank 27: 117131.52 + rank 28: 117154.42 + rank 29: 117129.33 + rank 30: 117129.62 + rank 31: 117125.47 + rank 32: 61003.92 + rank 33: 61006.30 + rank 34: 61015.40 + rank 35: 61031.47 + rank 36: 61062.47 + rank 37: 61065.69 + rank 38: 61041.61 + rank 39: 61038.70 + rank 40: 61070.79 + rank 41: 61035.90 + rank 42: 61044.31 + rank 43: 61039.79 + rank 44: 61030.44 + rank 45: 61057.34 + rank 46: 61024.75 + rank 47: 61040.95 + rank 48: 61004.63 + rank 49: 61016.01 + rank 50: 61011.80 + rank 51: 60999.89 + rank 52: 61042.17 + rank 53: 61022.55 + rank 54: 60986.11 + rank 55: 61013.09 + rank 56: 61026.43 + rank 57: 61020.43 + rank 58: 61017.10 + rank 59: 60991.08 + rank 60: 61010.00 + rank 61: 61024.74 + rank 62: 61002.68 + rank 63: 61048.03 + backward-compute: + rank 0: 102448.45 + rank 1: 102442.23 + rank 2: 102475.30 + rank 3: 102484.66 + rank 4: 102488.46 + rank 5: 102479.96 + rank 6: 102471.77 + rank 7: 102466.25 + rank 8: 102527.79 + rank 9: 102514.18 + rank 10: 102524.03 + rank 11: 102525.38 + rank 12: 102510.88 + rank 13: 102512.29 + rank 14: 102508.96 + rank 15: 102513.02 + rank 16: 102461.02 + rank 17: 102460.68 + rank 18: 102465.49 + rank 19: 102462.39 + rank 20: 102471.63 + rank 21: 102467.53 + rank 22: 102466.09 + rank 23: 102467.38 + rank 24: 102527.06 + rank 25: 102530.44 + rank 26: 102530.83 + rank 27: 102528.87 + rank 28: 102502.23 + rank 29: 102525.94 + rank 30: 102547.23 + rank 31: 102541.89 + rank 32: 158713.67 + rank 33: 158708.42 + rank 34: 158695.88 + rank 35: 158681.69 + rank 36: 158655.88 + rank 37: 158649.42 + rank 38: 158672.80 + rank 39: 158680.25 + rank 40: 158655.80 + rank 41: 158686.70 + rank 42: 158679.53 + rank 43: 158681.11 + rank 44: 158689.09 + rank 45: 158664.72 + rank 46: 158698.59 + rank 47: 158686.25 + rank 48: 158727.39 + rank 49: 158713.39 + rank 50: 158717.67 + rank 51: 158729.42 + rank 52: 158691.17 + rank 53: 158707.78 + rank 54: 158746.98 + rank 55: 158721.69 + rank 56: 158710.75 + rank 57: 158712.53 + rank 58: 158712.97 + rank 59: 158738.48 + rank 60: 158720.73 + rank 61: 158707.02 + rank 62: 158733.67 + rank 63: 158688.39 + batch-generator: + rank 0: 34.56 + rank 1: 37.80 + rank 2: 38.03 + rank 3: 38.30 + rank 4: 39.60 + rank 5: 37.57 + rank 6: 39.18 + rank 7: 36.82 + rank 8: 62.53 + rank 9: 66.39 + rank 10: 61.71 + rank 11: 62.88 + rank 12: 64.22 + rank 13: 66.42 + rank 14: 68.30 + rank 15: 66.48 + rank 16: 44.82 + rank 17: 46.62 + rank 18: 46.25 + rank 19: 47.40 + rank 20: 46.51 + rank 21: 48.65 + rank 22: 47.08 + rank 23: 43.82 + rank 24: 45.61 + rank 25: 45.77 + rank 26: 51.98 + rank 27: 46.96 + rank 28: 49.04 + rank 29: 51.99 + rank 30: 44.11 + rank 31: 46.23 + rank 32: 44.52 + rank 33: 47.87 + rank 34: 47.93 + rank 35: 49.11 + rank 36: 46.57 + rank 37: 45.75 + rank 38: 42.00 + rank 39: 39.28 + rank 40: 32.89 + rank 41: 37.10 + rank 42: 35.24 + rank 43: 36.44 + rank 44: 36.22 + rank 45: 35.65 + rank 46: 37.28 + rank 47: 29.25 + rank 48: 41.93 + rank 49: 42.73 + rank 50: 44.55 + rank 51: 45.14 + rank 52: 38.61 + rank 53: 42.02 + rank 54: 35.30 + rank 55: 38.00 + rank 56: 31.42 + rank 57: 37.91 + rank 58: 39.07 + rank 59: 38.27 + rank 60: 38.36 + rank 61: 36.85 + rank 62: 35.12 + rank 63: 31.41 + embedding-grads-all-reduce: + rank 0: 0.03 + rank 1: 0.03 + rank 2: 0.03 + rank 3: 0.03 + rank 4: 0.03 + rank 5: 0.03 + rank 6: 0.03 + rank 7: 0.03 + rank 8: 0.03 + rank 9: 0.03 + rank 10: 0.03 + rank 11: 0.03 + rank 12: 0.03 + rank 13: 0.03 + rank 14: 0.03 + rank 15: 0.03 + rank 16: 0.03 + rank 17: 0.03 + rank 18: 0.03 + rank 19: 0.03 + rank 20: 0.03 + rank 21: 0.03 + rank 22: 0.03 + rank 23: 0.03 + rank 24: 0.04 + rank 25: 0.04 + rank 26: 0.04 + rank 27: 0.03 + rank 28: 0.04 + rank 29: 0.04 + rank 30: 0.04 + rank 31: 0.04 + rank 32: 0.03 + rank 33: 0.04 + rank 34: 0.04 + rank 35: 0.03 + rank 36: 0.03 + rank 37: 0.03 + rank 38: 0.03 + rank 39: 0.04 + rank 40: 0.03 + rank 41: 0.02 + rank 42: 0.03 + rank 43: 0.03 + rank 44: 0.03 + rank 45: 0.03 + rank 46: 0.03 + rank 47: 0.03 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.03 + rank 57: 0.03 + rank 58: 0.03 + rank 59: 0.03 + rank 60: 0.03 + rank 61: 0.03 + rank 62: 0.03 + rank 63: 0.03 + all-grads-sync: + rank 0: 0.86 + rank 1: 0.87 + rank 2: 1.03 + rank 3: 1.66 + rank 4: 0.86 + rank 5: 0.87 + rank 6: 0.82 + rank 7: 0.88 + rank 8: 0.86 + rank 9: 1.09 + rank 10: 1.55 + rank 11: 1.01 + rank 12: 0.83 + rank 13: 0.86 + rank 14: 1.03 + rank 15: 0.79 + rank 16: 1.15 + rank 17: 1.26 + rank 18: 0.97 + rank 19: 0.86 + rank 20: 1.14 + rank 21: 0.84 + rank 22: 0.88 + rank 23: 0.81 + rank 24: 1.30 + rank 25: 1.42 + rank 26: 1.46 + rank 27: 1.22 + rank 28: 1.30 + rank 29: 1.43 + rank 30: 1.17 + rank 31: 1.27 + rank 32: 1.47 + rank 33: 1.44 + rank 34: 1.52 + rank 35: 1.78 + rank 36: 1.55 + rank 37: 1.47 + rank 38: 1.28 + rank 39: 1.19 + rank 40: 0.85 + rank 41: 0.84 + rank 42: 0.85 + rank 43: 0.87 + rank 44: 0.83 + rank 45: 1.11 + rank 46: 1.02 + rank 47: 0.86 + rank 48: 1.15 + rank 49: 1.09 + rank 50: 0.90 + rank 51: 1.45 + rank 52: 0.87 + rank 53: 1.11 + rank 54: 1.14 + rank 55: 1.06 + rank 56: 0.95 + rank 57: 0.94 + rank 58: 0.85 + rank 59: 0.86 + rank 60: 0.80 + rank 61: 1.16 + rank 62: 0.99 + rank 63: 0.97 + params-all-gather: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.03 + rank 5: 0.02 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.02 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.02 + rank 20: 0.02 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.02 + rank 24: 0.04 + rank 25: 0.04 + rank 26: 0.03 + rank 27: 0.04 + rank 28: 0.04 + rank 29: 0.03 + rank 30: 0.03 + rank 31: 0.04 + rank 32: 0.02 + rank 33: 0.02 + rank 34: 0.02 + rank 35: 0.02 + rank 36: 0.02 + rank 37: 0.02 + rank 38: 0.03 + rank 39: 0.03 + rank 40: 0.02 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.02 + rank 45: 0.02 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.02 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.02 + optimizer-copy-to-main-grad: + rank 0: 2.22 + rank 1: 1.81 + rank 2: 1.75 + rank 3: 2.16 + rank 4: 1.71 + rank 5: 1.92 + rank 6: 1.90 + rank 7: 1.91 + rank 8: 1.89 + rank 9: 2.14 + rank 10: 1.79 + rank 11: 1.81 + rank 12: 1.88 + rank 13: 1.83 + rank 14: 1.91 + rank 15: 2.12 + rank 16: 2.47 + rank 17: 2.20 + rank 18: 2.27 + rank 19: 2.18 + rank 20: 2.21 + rank 21: 2.35 + rank 22: 2.61 + rank 23: 2.33 + rank 24: 1.98 + rank 25: 2.28 + rank 26: 2.03 + rank 27: 2.31 + rank 28: 2.51 + rank 29: 2.50 + rank 30: 2.26 + rank 31: 2.31 + rank 32: 2.08 + rank 33: 1.96 + rank 34: 2.00 + rank 35: 2.25 + rank 36: 2.09 + rank 37: 2.03 + rank 38: 2.31 + rank 39: 2.00 + rank 40: 1.77 + rank 41: 1.90 + rank 42: 1.90 + rank 43: 1.93 + rank 44: 1.86 + rank 45: 2.07 + rank 46: 1.75 + rank 47: 1.67 + rank 48: 1.95 + rank 49: 1.82 + rank 50: 2.48 + rank 51: 1.82 + rank 52: 1.95 + rank 53: 1.89 + rank 54: 1.86 + rank 55: 1.86 + rank 56: 1.81 + rank 57: 2.39 + rank 58: 1.82 + rank 59: 1.81 + rank 60: 1.78 + rank 61: 1.73 + rank 62: 1.87 + rank 63: 1.77 + optimizer-inner-step: + rank 0: 43.96 + rank 1: 43.85 + rank 2: 43.59 + rank 3: 43.71 + rank 4: 43.58 + rank 5: 43.75 + rank 6: 44.09 + rank 7: 43.95 + rank 8: 43.85 + rank 9: 43.86 + rank 10: 43.60 + rank 11: 43.79 + rank 12: 43.84 + rank 13: 44.01 + rank 14: 43.82 + rank 15: 43.63 + rank 16: 44.18 + rank 17: 43.89 + rank 18: 44.00 + rank 19: 43.87 + rank 20: 43.84 + rank 21: 43.74 + rank 22: 44.32 + rank 23: 44.01 + rank 24: 39.82 + rank 25: 44.24 + rank 26: 41.62 + rank 27: 45.42 + rank 28: 49.36 + rank 29: 44.39 + rank 30: 43.73 + rank 31: 45.42 + rank 32: 43.90 + rank 33: 43.74 + rank 34: 43.88 + rank 35: 48.74 + rank 36: 43.89 + rank 37: 43.83 + rank 38: 43.79 + rank 39: 43.74 + rank 40: 43.75 + rank 41: 43.93 + rank 42: 43.91 + rank 43: 44.02 + rank 44: 43.60 + rank 45: 44.23 + rank 46: 43.85 + rank 47: 43.53 + rank 48: 43.76 + rank 49: 43.59 + rank 50: 54.64 + rank 51: 43.47 + rank 52: 43.85 + rank 53: 43.65 + rank 54: 43.84 + rank 55: 43.75 + rank 56: 43.49 + rank 57: 46.59 + rank 58: 43.72 + rank 59: 43.61 + rank 60: 43.92 + rank 61: 43.61 + rank 62: 43.73 + rank 63: 43.50 + optimizer-copy-main-to-model-params: + rank 0: 4.46 + rank 1: 3.85 + rank 2: 3.77 + rank 3: 4.38 + rank 4: 3.69 + rank 5: 4.04 + rank 6: 3.98 + rank 7: 4.02 + rank 8: 3.92 + rank 9: 4.44 + rank 10: 3.84 + rank 11: 3.83 + rank 12: 3.94 + rank 13: 3.86 + rank 14: 3.97 + rank 15: 4.21 + rank 16: 4.46 + rank 17: 4.01 + rank 18: 4.07 + rank 19: 4.05 + rank 20: 4.03 + rank 21: 4.23 + rank 22: 4.60 + rank 23: 4.22 + rank 24: 3.94 + rank 25: 4.22 + rank 26: 3.94 + rank 27: 4.12 + rank 28: 4.42 + rank 29: 4.38 + rank 30: 4.42 + rank 31: 4.04 + rank 32: 4.13 + rank 33: 3.92 + rank 34: 4.01 + rank 35: 4.54 + rank 36: 4.21 + rank 37: 4.09 + rank 38: 4.49 + rank 39: 3.92 + rank 40: 3.75 + rank 41: 4.00 + rank 42: 3.91 + rank 43: 3.95 + rank 44: 3.96 + rank 45: 4.27 + rank 46: 3.77 + rank 47: 3.74 + rank 48: 4.11 + rank 49: 3.89 + rank 50: 4.96 + rank 51: 4.05 + rank 52: 4.19 + rank 53: 3.90 + rank 54: 3.92 + rank 55: 3.93 + rank 56: 3.77 + rank 57: 4.67 + rank 58: 3.88 + rank 59: 3.75 + rank 60: 3.77 + rank 61: 3.64 + rank 62: 3.93 + rank 63: 3.66 + optimizer: + rank 0: 55.88 + rank 1: 54.94 + rank 2: 54.50 + rank 3: 55.51 + rank 4: 54.44 + rank 5: 55.06 + rank 6: 55.32 + rank 7: 55.29 + rank 8: 54.97 + rank 9: 55.70 + rank 10: 54.63 + rank 11: 54.78 + rank 12: 54.98 + rank 13: 55.07 + rank 14: 55.01 + rank 15: 55.16 + rank 16: 56.26 + rank 17: 55.34 + rank 18: 55.49 + rank 19: 55.35 + rank 20: 55.29 + rank 21: 55.49 + rank 22: 56.59 + rank 23: 55.78 + rank 24: 54.69 + rank 25: 59.44 + rank 26: 56.50 + rank 27: 60.46 + rank 28: 64.87 + rank 29: 59.87 + rank 30: 59.20 + rank 31: 60.45 + rank 32: 55.24 + rank 33: 54.87 + rank 34: 55.04 + rank 35: 60.65 + rank 36: 55.42 + rank 37: 55.22 + rank 38: 55.62 + rank 39: 54.84 + rank 40: 54.66 + rank 41: 55.11 + rank 42: 55.04 + rank 43: 55.17 + rank 44: 54.79 + rank 45: 55.81 + rank 46: 54.75 + rank 47: 54.39 + rank 48: 55.15 + rank 49: 54.73 + rank 50: 67.31 + rank 51: 54.80 + rank 52: 55.35 + rank 53: 54.78 + rank 54: 54.96 + rank 55: 54.91 + rank 56: 54.43 + rank 57: 58.83 + rank 58: 54.81 + rank 59: 54.49 + rank 60: 54.82 + rank 61: 54.37 + rank 62: 54.85 + rank 63: 54.31 + interval-time: + rank 0: 0.99 + rank 1: 0.97 + rank 2: 1.01 + rank 3: 0.92 + rank 4: 0.99 + rank 5: 1.04 + rank 6: 0.93 + rank 7: 0.91 + rank 8: 0.76 + rank 9: 0.86 + rank 10: 0.86 + rank 11: 0.85 + rank 12: 0.85 + rank 13: 0.75 + rank 14: 0.75 + rank 15: 0.83 + rank 16: 0.84 + rank 17: 0.83 + rank 18: 0.80 + rank 19: 0.86 + rank 20: 0.73 + rank 21: 0.83 + rank 22: 0.74 + rank 23: 0.82 + rank 24: 0.95 + rank 25: 0.95 + rank 26: 0.90 + rank 27: 0.97 + rank 28: 0.92 + rank 29: 0.95 + rank 30: 0.99 + rank 31: 0.96 + rank 32: 0.90 + rank 33: 1.00 + rank 34: 0.92 + rank 35: 1.08 + rank 36: 1.20 + rank 37: 0.96 + rank 38: 0.88 + rank 39: 0.86 + rank 40: 0.83 + rank 41: 0.80 + rank 42: 0.83 + rank 43: 0.74 + rank 44: 0.73 + rank 45: 0.82 + rank 46: 0.74 + rank 47: 0.80 + rank 48: 0.92 + rank 49: 0.99 + rank 50: 0.86 + rank 51: 0.94 + rank 52: 0.81 + rank 53: 0.98 + rank 54: 0.92 + rank 55: 0.91 + rank 56: 0.84 + rank 57: 0.81 + rank 58: 0.82 + rank 59: 0.74 + rank 60: 0.73 + rank 61: 0.81 + rank 62: 0.72 + rank 63: 2.44 + [2026-10-16 19:56:33] iteration 2006/ 2384 | consumed samples: 64192 | elapsed time per iteration (ms): 220631.3 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.65% | FLOPs estimator: workload-aware | logical tokens/s: 76041.8 | valid tokens/s: 76041.8 | physical tokens/s: 77329.8 | loss tokens/s: 56171.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.658300E-01 | lm-global-supervised-tokens: 1.239309E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.332965E-02 | lm-token-buffer-mib: 2.329936E+02 | global_load_balancing_loss: 9.987928E-01 | loss scale: 1.0 | grad norm: 0.273 | params norm: 11152.083 | params rms: 0.060385 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:09:58.613906 | finish at 2026-10-17 19:06:31 + [2026-10-16 20:00:15] iteration 2007/ 2384 | consumed samples: 64224 | elapsed time per iteration (ms): 222440.7 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.08% | FLOPs estimator: workload-aware | logical tokens/s: 75423.3 | valid tokens/s: 75423.3 | physical tokens/s: 75784.1 | loss tokens/s: 56817.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.635548E-01 | lm-global-supervised-tokens: 1.263862E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.411253E-02 | lm-token-buffer-mib: 2.331727E+02 | global_load_balancing_loss: 1.000536E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11152.055 | params rms: 0.060385 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:17:40.129000 | finish at 2026-10-17 19:17:55 + [2026-10-16 20:04:04] iteration 2008/ 2384 | consumed samples: 64256 | elapsed time per iteration (ms): 228477.8 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.13% | FLOPs estimator: workload-aware | logical tokens/s: 73430.3 | valid tokens/s: 73430.3 | physical tokens/s: 74370.2 | loss tokens/s: 55437.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.918694E-01 | lm-global-supervised-tokens: 1.266615E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.397353E-02 | lm-token-buffer-mib: 2.362801E+02 | global_load_balancing_loss: 9.996861E-01 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11152.027 | params rms: 0.060385 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:51:47.646484 | finish at 2026-10-17 19:55:51 + [2026-10-16 20:07:48] iteration 2009/ 2384 | consumed samples: 64288 | elapsed time per iteration (ms): 223733.5 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.95% | FLOPs estimator: workload-aware | logical tokens/s: 74987.4 | valid tokens/s: 74987.4 | physical tokens/s: 75607.9 | loss tokens/s: 57171.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.869603E-01 | lm-global-supervised-tokens: 1.279107E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.356114E-02 | lm-token-buffer-mib: 2.346129E+02 | global_load_balancing_loss: 9.998631E-01 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11152.000 | params rms: 0.060384 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:18:20.077790 | finish at 2026-10-17 19:26:08 + [2026-10-16 20:11:52] iteration 2010/ 2384 | consumed samples: 64320 | elapsed time per iteration (ms): 244221.3 | throughput per GPU (TFLOP/s/GPU): 19.1 | MFU 6.12% | FLOPs estimator: workload-aware | logical tokens/s: 68696.7 | valid tokens/s: 68696.7 | physical tokens/s: 69480.6 | loss tokens/s: 53134.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.039656E-01 | lm-global-supervised-tokens: 1.297666E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.515632E-02 | lm-token-buffer-mib: 2.355589E+02 | global_load_balancing_loss: 1.001548E+00 | loss scale: 1.0 | grad norm: 0.339 | params norm: 11151.971 | params rms: 0.060384 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:22:18.777125 | finish at 2026-10-17 21:34:11 + [2026-10-16 20:15:32] iteration 2011/ 2384 | consumed samples: 64352 | elapsed time per iteration (ms): 219872.1 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.66% | FLOPs estimator: workload-aware | logical tokens/s: 76304.3 | valid tokens/s: 76304.3 | physical tokens/s: 77091.5 | loss tokens/s: 61608.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.524560E-01 | lm-global-supervised-tokens: 1.354590E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.547320E-02 | lm-token-buffer-mib: 2.342681E+02 | global_load_balancing_loss: 1.002554E+00 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11151.940 | params rms: 0.060384 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:46:52.301524 | finish at 2026-10-17 19:02:24 + [2026-10-16 20:19:17] iteration 2012/ 2384 | consumed samples: 64384 | elapsed time per iteration (ms): 225683.5 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.27% | FLOPs estimator: workload-aware | logical tokens/s: 74339.5 | valid tokens/s: 74339.5 | physical tokens/s: 75149.2 | loss tokens/s: 56449.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.433022E-01 | lm-global-supervised-tokens: 1.273971E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.261920E-02 | lm-token-buffer-mib: 2.329218E+02 | global_load_balancing_loss: 1.001466E+00 | loss scale: 1.0 | grad norm: 0.452 | params norm: 11151.909 | params rms: 0.060384 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:19:14.269558 | finish at 2026-10-17 19:38:32 + [2026-10-16 20:23:01] iteration 2013/ 2384 | consumed samples: 64416 | elapsed time per iteration (ms): 223375.8 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.39% | FLOPs estimator: workload-aware | logical tokens/s: 75107.5 | valid tokens/s: 75107.4 | physical tokens/s: 75672.9 | loss tokens/s: 57800.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.947007E-01 | lm-global-supervised-tokens: 1.291121E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.213246E-02 | lm-token-buffer-mib: 2.346796E+02 | global_load_balancing_loss: 1.000452E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11151.878 | params rms: 0.060384 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:01:12.411412 | finish at 2026-10-17 19:24:13 + [2026-10-16 20:26:44] iteration 2014/ 2384 | consumed samples: 64448 | elapsed time per iteration (ms): 223303.1 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.85% | FLOPs estimator: workload-aware | logical tokens/s: 75132.0 | valid tokens/s: 75132.0 | physical tokens/s: 75921.1 | loss tokens/s: 58348.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.843124E-01 | lm-global-supervised-tokens: 1.302938E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.329670E-02 | lm-token-buffer-mib: 2.352472E+02 | global_load_balancing_loss: 9.998767E-01 | loss scale: 1.0 | grad norm: 0.301 | params norm: 11151.850 | params rms: 0.060384 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:57:02.154803 | finish at 2026-10-17 19:23:46 + [2026-10-16 20:30:28] iteration 2015/ 2384 | consumed samples: 64480 | elapsed time per iteration (ms): 224137.1 | throughput per GPU (TFLOP/s/GPU): 19.1 | MFU 6.12% | FLOPs estimator: workload-aware | logical tokens/s: 74852.4 | valid tokens/s: 74852.4 | physical tokens/s: 75366.7 | loss tokens/s: 55221.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.000460E-01 | lm-global-supervised-tokens: 1.237725E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.106906E-02 | lm-token-buffer-mib: 2.325376E+02 | global_load_balancing_loss: 9.995071E-01 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11151.822 | params rms: 0.060383 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:58:26.598339 | finish at 2026-10-17 19:28:55 + [2026-10-16 20:34:10] iteration 2016/ 2384 | consumed samples: 64512 | elapsed time per iteration (ms): 221419.4 | throughput per GPU (TFLOP/s/GPU): 23.7 | MFU 7.59% | FLOPs estimator: workload-aware | logical tokens/s: 75771.2 | valid tokens/s: 75771.2 | physical tokens/s: 76120.7 | loss tokens/s: 52249.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.017988E-01 | lm-global-supervised-tokens: 1.156894E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.388206E-02 | lm-token-buffer-mib: 2.334011E+02 | global_load_balancing_loss: 9.986477E-01 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11151.793 | params rms: 0.060383 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:38:02.321117 | finish at 2026-10-17 19:12:12 + [2026-10-16 20:37:51] iteration 2017/ 2384 | consumed samples: 64544 | elapsed time per iteration (ms): 221912.3 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.68% | FLOPs estimator: workload-aware | logical tokens/s: 75602.8 | valid tokens/s: 75602.8 | physical tokens/s: 76266.5 | loss tokens/s: 56835.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.275857E-01 | lm-global-supervised-tokens: 1.261256E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.287783E-02 | lm-token-buffer-mib: 2.337949E+02 | global_load_balancing_loss: 9.982844E-01 | loss scale: 1.0 | grad norm: 0.105 | params norm: 11151.760 | params rms: 0.060383 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:37:21.819565 | finish at 2026-10-17 19:15:13 + [2026-10-16 20:41:33] iteration 2018/ 2384 | consumed samples: 64576 | elapsed time per iteration (ms): 221303.6 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.61% | FLOPs estimator: workload-aware | logical tokens/s: 75810.8 | valid tokens/s: 75810.8 | physical tokens/s: 76489.0 | loss tokens/s: 58386.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.042849E-01 | lm-global-supervised-tokens: 1.292119E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.350210E-02 | lm-token-buffer-mib: 2.341765E+02 | global_load_balancing_loss: 9.991714E-01 | loss scale: 1.0 | grad norm: 0.157 | params norm: 11151.729 | params rms: 0.060383 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:29:57.122077 | finish at 2026-10-17 19:11:30 + [2026-10-16 20:45:14] iteration 2019/ 2384 | consumed samples: 64608 | elapsed time per iteration (ms): 220961.5 | throughput per GPU (TFLOP/s/GPU): 19.2 | MFU 6.15% | FLOPs estimator: workload-aware | logical tokens/s: 75928.1 | valid tokens/s: 75928.1 | physical tokens/s: 76534.2 | loss tokens/s: 57585.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.127025E-01 | lm-global-supervised-tokens: 1.272412E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.506165E-02 | lm-token-buffer-mib: 2.346299E+02 | global_load_balancing_loss: 1.001344E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11151.697 | params rms: 0.060383 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:24:10.951651 | finish at 2026-10-17 19:09:25 + [2026-10-16 20:49:16] iteration 2020/ 2384 | consumed samples: 64640 | elapsed time per iteration (ms): 242167.0 | throughput per GPU (TFLOP/s/GPU): 17.5 | MFU 5.62% | FLOPs estimator: workload-aware | logical tokens/s: 69279.5 | valid tokens/s: 69279.5 | physical tokens/s: 70211.1 | loss tokens/s: 54701.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.329467E-01 | lm-global-supervised-tokens: 1.324684E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.349487E-02 | lm-token-buffer-mib: 2.356821E+02 | global_load_balancing_loss: 1.001128E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11151.666 | params rms: 0.060383 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:29:08.795657 | finish at 2026-10-17 21:18:25 + [2026-10-16 20:52:56] iteration 2021/ 2384 | consumed samples: 64672 | elapsed time per iteration (ms): 220215.3 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.41% | FLOPs estimator: workload-aware | logical tokens/s: 76185.3 | valid tokens/s: 76185.3 | physical tokens/s: 76621.5 | loss tokens/s: 58003.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.472034E-01 | lm-global-supervised-tokens: 1.277318E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.886706E-02 | lm-token-buffer-mib: 2.328904E+02 | global_load_balancing_loss: 1.000582E+00 | loss scale: 1.0 | grad norm: 0.158 | params norm: 11151.633 | params rms: 0.060382 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:12:18.143198 | finish at 2026-10-17 19:05:14 + [2026-10-16 20:56:41] iteration 2022/ 2384 | consumed samples: 64704 | elapsed time per iteration (ms): 224502.2 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.13% | FLOPs estimator: workload-aware | logical tokens/s: 74730.7 | valid tokens/s: 74730.7 | physical tokens/s: 75496.4 | loss tokens/s: 56577.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.071619E-01 | lm-global-supervised-tokens: 1.270173E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.584049E-02 | lm-token-buffer-mib: 2.328270E+02 | global_load_balancing_loss: 1.002181E+00 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11151.598 | params rms: 0.060382 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:34:29.807062 | finish at 2026-10-17 19:31:10 + [2026-10-16 21:00:23] iteration 2023/ 2384 | consumed samples: 64736 | elapsed time per iteration (ms): 222510.1 | throughput per GPU (TFLOP/s/GPU): 19.6 | MFU 6.29% | FLOPs estimator: workload-aware | logical tokens/s: 75399.7 | valid tokens/s: 75399.7 | physical tokens/s: 76082.3 | loss tokens/s: 58662.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.115098E-01 | lm-global-supervised-tokens: 1.305303E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.707444E-02 | lm-token-buffer-mib: 2.339582E+02 | global_load_balancing_loss: 1.001681E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11151.566 | params rms: 0.060382 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:18:46.162068 | finish at 2026-10-17 19:19:09 + [2026-10-16 21:04:05] iteration 2024/ 2384 | consumed samples: 64768 | elapsed time per iteration (ms): 221750.7 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.18% | FLOPs estimator: workload-aware | logical tokens/s: 75657.9 | valid tokens/s: 75657.9 | physical tokens/s: 76293.2 | loss tokens/s: 52930.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.132396E-01 | lm-global-supervised-tokens: 1.173737E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.306041E-02 | lm-token-buffer-mib: 2.339361E+02 | global_load_balancing_loss: 1.001736E+00 | loss scale: 1.0 | grad norm: 0.112 | params norm: 11151.535 | params rms: 0.060382 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:10:30.255346 | finish at 2026-10-17 19:14:35 + [2026-10-16 21:07:45] iteration 2025/ 2384 | consumed samples: 64800 | elapsed time per iteration (ms): 220361.6 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.77% | FLOPs estimator: workload-aware | logical tokens/s: 76134.8 | valid tokens/s: 76134.8 | physical tokens/s: 76904.9 | loss tokens/s: 55492.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.649534E-01 | lm-global-supervised-tokens: 1.222832E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.658630E-02 | lm-token-buffer-mib: 2.343761E+02 | global_load_balancing_loss: 9.991602E-01 | loss scale: 1.0 | grad norm: 0.096 | params norm: 11151.502 | params rms: 0.060382 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:58:29.826184 | finish at 2026-10-17 19:06:15 + [2026-10-16 21:11:32] iteration 2026/ 2384 | consumed samples: 64832 | elapsed time per iteration (ms): 226407.2 | throughput per GPU (TFLOP/s/GPU): 23.9 | MFU 7.66% | FLOPs estimator: workload-aware | logical tokens/s: 74101.7 | valid tokens/s: 74101.7 | physical tokens/s: 74315.9 | loss tokens/s: 52600.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.932001E-01 | lm-global-supervised-tokens: 1.190919E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.948985E-02 | lm-token-buffer-mib: 2.331127E+02 | global_load_balancing_loss: 9.982187E-01 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11151.470 | params rms: 0.060382 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:30:53.779684 | finish at 2026-10-17 19:42:25 + [2026-10-16 21:15:12] iteration 2027/ 2384 | consumed samples: 64864 | elapsed time per iteration (ms): 220004.6 | throughput per GPU (TFLOP/s/GPU): 23.7 | MFU 7.60% | FLOPs estimator: workload-aware | logical tokens/s: 76258.3 | valid tokens/s: 76258.3 | physical tokens/s: 77282.2 | loss tokens/s: 56491.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.474447E-01 | lm-global-supervised-tokens: 1.242848E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.285291E-02 | lm-token-buffer-mib: 2.331876E+02 | global_load_balancing_loss: 9.987704E-01 | loss scale: 1.0 | grad norm: 0.127 | params norm: 11151.437 | params rms: 0.060381 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:49:01.626726 | finish at 2026-10-17 19:04:13 + [2026-10-16 21:18:54] iteration 2028/ 2384 | consumed samples: 64896 | elapsed time per iteration (ms): 222434.9 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.11% | FLOPs estimator: workload-aware | logical tokens/s: 75425.2 | valid tokens/s: 75425.2 | physical tokens/s: 76205.6 | loss tokens/s: 58728.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.417309E-01 | lm-global-supervised-tokens: 1.306327E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.138757E-02 | lm-token-buffer-mib: 2.348121E+02 | global_load_balancing_loss: 9.995650E-01 | loss scale: 1.0 | grad norm: 0.110 | params norm: 11151.407 | params rms: 0.060381 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:59:46.809053 | finish at 2026-10-17 19:18:41 + [2026-10-16 21:22:40] iteration 2029/ 2384 | consumed samples: 64928 | elapsed time per iteration (ms): 225800.4 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.85% | FLOPs estimator: workload-aware | logical tokens/s: 74301.1 | valid tokens/s: 74301.1 | physical tokens/s: 74923.8 | loss tokens/s: 54351.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.116733E-01 | lm-global-supervised-tokens: 1.227254E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.609442E-02 | lm-token-buffer-mib: 2.347682E+02 | global_load_balancing_loss: 1.000701E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11151.376 | params rms: 0.060381 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:15:59.126179 | finish at 2026-10-17 19:38:39 + [2026-10-16 21:26:48] iteration 2030/ 2384 | consumed samples: 64960 | elapsed time per iteration (ms): 247695.0 | throughput per GPU (TFLOP/s/GPU): 18.3 | MFU 5.85% | FLOPs estimator: workload-aware | logical tokens/s: 67733.3 | valid tokens/s: 67733.3 | physical tokens/s: 68753.2 | loss tokens/s: 52258.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.451287E-01 | lm-global-supervised-tokens: 1.294410E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.335961E-02 | lm-token-buffer-mib: 2.373069E+02 | global_load_balancing_loss: 1.001759E+00 | loss scale: 1.0 | grad norm: 0.180 | params norm: 11151.345 | params rms: 0.060381 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:21:24.023984 | finish at 2026-10-17 21:48:12 + [2026-10-16 21:30:27] iteration 2031/ 2384 | consumed samples: 64992 | elapsed time per iteration (ms): 219480.1 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.94% | FLOPs estimator: workload-aware | logical tokens/s: 76440.6 | valid tokens/s: 76440.6 | physical tokens/s: 76915.7 | loss tokens/s: 59774.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.957730E-01 | lm-global-supervised-tokens: 1.311930E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.681456E-02 | lm-token-buffer-mib: 2.326885E+02 | global_load_balancing_loss: 1.000590E+00 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11151.312 | params rms: 0.060381 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:31:16.486212 | finish at 2026-10-17 19:01:44 + [2026-10-16 21:34:14] iteration 2032/ 2384 | consumed samples: 65024 | elapsed time per iteration (ms): 226596.6 | throughput per GPU (TFLOP/s/GPU): 25.3 | MFU 8.11% | FLOPs estimator: workload-aware | logical tokens/s: 74039.9 | valid tokens/s: 74039.9 | physical tokens/s: 74351.5 | loss tokens/s: 59212.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.745246E-01 | lm-global-supervised-tokens: 1.341733E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.559575E-02 | lm-token-buffer-mib: 2.330109E+02 | global_load_balancing_loss: 1.002413E+00 | loss scale: 1.0 | grad norm: 0.160 | params norm: 11151.276 | params rms: 0.060381 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:09:22.004059 | finish at 2026-10-17 19:43:36 + [2026-10-16 21:37:57] iteration 2033/ 2384 | consumed samples: 65056 | elapsed time per iteration (ms): 223501.7 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.42% | FLOPs estimator: workload-aware | logical tokens/s: 75065.2 | valid tokens/s: 75065.2 | physical tokens/s: 76375.6 | loss tokens/s: 56380.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.255534E-01 | lm-global-supervised-tokens: 1.260106E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.866888E-02 | lm-token-buffer-mib: 2.366571E+02 | global_load_balancing_loss: 1.002511E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11151.245 | params rms: 0.060380 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:47:29.094498 | finish at 2026-10-17 19:25:26 + [2026-10-16 21:41:38] iteration 2034/ 2384 | consumed samples: 65088 | elapsed time per iteration (ms): 220461.7 | throughput per GPU (TFLOP/s/GPU): 23.2 | MFU 7.45% | FLOPs estimator: workload-aware | logical tokens/s: 76100.2 | valid tokens/s: 76100.2 | physical tokens/s: 76773.9 | loss tokens/s: 58628.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.195698E-01 | lm-global-supervised-tokens: 1.292535E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.518681E-02 | lm-token-buffer-mib: 2.330536E+02 | global_load_balancing_loss: 9.993323E-01 | loss scale: 1.0 | grad norm: 0.192 | params norm: 11151.215 | params rms: 0.060380 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:26:01.586142 | finish at 2026-10-17 19:07:39 + [2026-10-16 21:45:20] iteration 2035/ 2384 | consumed samples: 65120 | elapsed time per iteration (ms): 222464.1 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.33% | FLOPs estimator: workload-aware | logical tokens/s: 75415.4 | valid tokens/s: 75415.4 | physical tokens/s: 75840.0 | loss tokens/s: 53641.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.036257E-01 | lm-global-supervised-tokens: 1.193337E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.521851E-02 | lm-token-buffer-mib: 2.333511E+02 | global_load_balancing_loss: 9.987119E-01 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11151.185 | params rms: 0.060380 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:33:59.966949 | finish at 2026-10-17 19:19:20 + [2026-10-16 21:49:02] iteration 2036/ 2384 | consumed samples: 65152 | elapsed time per iteration (ms): 221688.2 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.96% | FLOPs estimator: workload-aware | logical tokens/s: 75679.3 | valid tokens/s: 75679.3 | physical tokens/s: 76188.6 | loss tokens/s: 53710.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.484664E-01 | lm-global-supervised-tokens: 1.190700E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.276187E-02 | lm-token-buffer-mib: 2.326320E+02 | global_load_balancing_loss: 9.996180E-01 | loss scale: 1.0 | grad norm: 0.132 | params norm: 11151.151 | params rms: 0.060380 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:25:47.498992 | finish at 2026-10-17 19:14:49 + [2026-10-16 21:52:44] iteration 2037/ 2384 | consumed samples: 65184 | elapsed time per iteration (ms): 222260.8 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.07% | FLOPs estimator: workload-aware | logical tokens/s: 75484.3 | valid tokens/s: 75484.3 | physical tokens/s: 76263.9 | loss tokens/s: 56061.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.566006E-01 | lm-global-supervised-tokens: 1.246024E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.603526E-02 | lm-token-buffer-mib: 2.330247E+02 | global_load_balancing_loss: 9.995747E-01 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11151.118 | params rms: 0.060380 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:25:24.493175 | finish at 2026-10-17 19:18:09 + [2026-10-16 21:56:27] iteration 2038/ 2384 | consumed samples: 65216 | elapsed time per iteration (ms): 223003.2 | throughput per GPU (TFLOP/s/GPU): 27.8 | MFU 8.89% | FLOPs estimator: workload-aware | logical tokens/s: 75232.8 | valid tokens/s: 75232.8 | physical tokens/s: 75558.5 | loss tokens/s: 59439.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.043201E-01 | lm-global-supervised-tokens: 1.325514E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.475038E-02 | lm-token-buffer-mib: 2.327056E+02 | global_load_balancing_loss: 9.998572E-01 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11151.084 | params rms: 0.060379 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:25:59.114726 | finish at 2026-10-17 19:22:26 + [2026-10-16 22:00:11] iteration 2039/ 2384 | consumed samples: 65248 | elapsed time per iteration (ms): 223965.1 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.82% | FLOPs estimator: workload-aware | logical tokens/s: 74909.9 | valid tokens/s: 74909.9 | physical tokens/s: 75950.7 | loss tokens/s: 52975.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.029627E-01 | lm-global-supervised-tokens: 1.186456E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.430937E-02 | lm-token-buffer-mib: 2.359153E+02 | global_load_balancing_loss: 1.000035E+00 | loss scale: 1.0 | grad norm: 0.281 | params norm: 11151.049 | params rms: 0.060379 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:27:47.952937 | finish at 2026-10-17 19:27:59 + [2026-10-16 22:04:15] iteration 2040/ 2384 | consumed samples: 65280 | elapsed time per iteration (ms): 244350.1 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.95% | FLOPs estimator: workload-aware | logical tokens/s: 68660.5 | valid tokens/s: 68660.5 | physical tokens/s: 69209.8 | loss tokens/s: 51292.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.746301E-01 | lm-global-supervised-tokens: 1.253334E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.523080E-02 | lm-token-buffer-mib: 2.333687E+02 | global_load_balancing_loss: 1.001237E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11151.016 | params rms: 0.060379 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:20:56.422276 | finish at 2026-10-17 21:25:12 +times across ranks (ms): + save-checkpoint: + rank 0: 24169.52 + rank 1: 24169.52 + rank 2: 24169.53 + rank 3: 24169.55 + rank 4: 24169.51 + rank 5: 24169.53 + rank 6: 24169.54 + rank 7: 24169.52 + rank 8: 24169.69 + rank 9: 24169.71 + rank 10: 24169.71 + rank 11: 24169.68 + rank 12: 24169.71 + rank 13: 24169.69 + rank 14: 24169.71 + rank 15: 24169.70 + rank 16: 24170.98 + rank 17: 24170.99 + rank 18: 24170.97 + rank 19: 24170.98 + rank 20: 24170.95 + rank 21: 24170.97 + rank 22: 24170.98 + rank 23: 24170.95 + rank 24: 24169.96 + rank 25: 24169.96 + rank 26: 24169.98 + rank 27: 24169.99 + rank 28: 24169.98 + rank 29: 24170.00 + rank 30: 24169.98 + rank 31: 24170.05 + rank 32: 24168.79 + rank 33: 24168.80 + rank 34: 24168.78 + rank 35: 24168.79 + rank 36: 24168.76 + rank 37: 24168.80 + rank 38: 24168.76 + rank 39: 24168.79 + rank 40: 24168.94 + rank 41: 24168.97 + rank 42: 24168.93 + rank 43: 24168.93 + rank 44: 24168.94 + rank 45: 24168.95 + rank 46: 24168.96 + rank 47: 24168.96 + rank 48: 24169.10 + rank 49: 24169.14 + rank 50: 24169.13 + rank 51: 24169.09 + rank 52: 24169.09 + rank 53: 24169.12 + rank 54: 24169.09 + rank 55: 24169.14 + rank 56: 24169.82 + rank 57: 24169.81 + rank 58: 24169.81 + rank 59: 24169.81 + rank 60: 24169.86 + rank 61: 24169.81 + rank 62: 24169.79 + rank 63: 24169.81 + [2026-10-16 22:08:22] iteration 2041/ 2384 | consumed samples: 65312 | elapsed time per iteration (ms): 221715.4 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.80% | FLOPs estimator: workload-aware | logical tokens/s: 75670.0 | valid tokens/s: 75670.0 | physical tokens/s: 76438.8 | loss tokens/s: 58264.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.395319E-01 | lm-global-supervised-tokens: 1.291806E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.575032E-02 | lm-token-buffer-mib: 2.337086E+02 | global_load_balancing_loss: 1.000841E+00 | loss scale: 1.0 | grad norm: 0.236 | params norm: 11150.982 | params rms: 0.060379 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:07:28.378350 | finish at 2026-10-17 19:15:50 + [2026-10-16 22:12:09] iteration 2042/ 2384 | consumed samples: 65344 | elapsed time per iteration (ms): 226909.8 | throughput per GPU (TFLOP/s/GPU): 19.3 | MFU 6.18% | FLOPs estimator: workload-aware | logical tokens/s: 73937.8 | valid tokens/s: 73937.8 | physical tokens/s: 74648.3 | loss tokens/s: 59211.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.847852E-01 | lm-global-supervised-tokens: 1.343568E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.546453E-02 | lm-token-buffer-mib: 2.327021E+02 | global_load_balancing_loss: 1.000122E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11150.947 | params rms: 0.060379 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:33:23.148846 | finish at 2026-10-17 19:45:32 + [2026-10-16 22:15:49] iteration 2043/ 2384 | consumed samples: 65376 | elapsed time per iteration (ms): 220714.5 | throughput per GPU (TFLOP/s/GPU): 18.9 | MFU 6.05% | FLOPs estimator: workload-aware | logical tokens/s: 76013.1 | valid tokens/s: 76013.1 | physical tokens/s: 76627.1 | loss tokens/s: 59131.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.727199E-01 | lm-global-supervised-tokens: 1.305119E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.771234E-02 | lm-token-buffer-mib: 2.336344E+02 | global_load_balancing_loss: 1.001357E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11150.912 | params rms: 0.060379 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:54:23.645947 | finish at 2026-10-17 19:10:13 + [2026-10-16 22:19:35] iteration 2044/ 2384 | consumed samples: 65408 | elapsed time per iteration (ms): 225465.7 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.04% | FLOPs estimator: workload-aware | logical tokens/s: 74411.2 | valid tokens/s: 74411.2 | physical tokens/s: 75857.4 | loss tokens/s: 54983.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.199663E-01 | lm-global-supervised-tokens: 1.239684E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.118598E-02 | lm-token-buffer-mib: 2.375754E+02 | global_load_balancing_loss: 1.001518E+00 | loss scale: 1.0 | grad norm: 0.371 | params norm: 11150.880 | params rms: 0.060378 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:17:38.342834 | finish at 2026-10-17 19:37:13 + [2026-10-16 22:23:16] iteration 2045/ 2384 | consumed samples: 65440 | elapsed time per iteration (ms): 221147.0 | throughput per GPU (TFLOP/s/GPU): 23.5 | MFU 7.53% | FLOPs estimator: workload-aware | logical tokens/s: 75864.4 | valid tokens/s: 75864.4 | physical tokens/s: 76313.7 | loss tokens/s: 55438.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.104108E-01 | lm-global-supervised-tokens: 1.226016E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.938587E-02 | lm-token-buffer-mib: 2.336009E+02 | global_load_balancing_loss: 1.001897E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11150.846 | params rms: 0.060378 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:49:28.849028 | finish at 2026-10-17 19:12:45 + [2026-10-16 22:26:57] iteration 2046/ 2384 | consumed samples: 65472 | elapsed time per iteration (ms): 220721.2 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.41% | FLOPs estimator: workload-aware | logical tokens/s: 76010.8 | valid tokens/s: 76010.7 | physical tokens/s: 76672.6 | loss tokens/s: 60551.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.604456E-01 | lm-global-supervised-tokens: 1.336492E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.300356E-02 | lm-token-buffer-mib: 2.338268E+02 | global_load_balancing_loss: 1.001051E+00 | loss scale: 1.0 | grad norm: 0.268 | params norm: 11150.808 | params rms: 0.060378 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:43:23.759552 | finish at 2026-10-17 19:10:20 + [2026-10-16 22:30:37] iteration 2047/ 2384 | consumed samples: 65504 | elapsed time per iteration (ms): 220244.9 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.85% | FLOPs estimator: workload-aware | logical tokens/s: 76175.3 | valid tokens/s: 76175.3 | physical tokens/s: 77105.7 | loss tokens/s: 56097.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.116555E-01 | lm-global-supervised-tokens: 1.235507E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.642164E-02 | lm-token-buffer-mib: 2.351888E+02 | global_load_balancing_loss: 9.994797E-01 | loss scale: 1.0 | grad norm: 0.262 | params norm: 11150.774 | params rms: 0.060378 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:37:02.527841 | finish at 2026-10-17 19:07:39 + [2026-10-16 22:34:21] iteration 2048/ 2384 | consumed samples: 65536 | elapsed time per iteration (ms): 224405.4 | throughput per GPU (TFLOP/s/GPU): 23.4 | MFU 7.50% | FLOPs estimator: workload-aware | logical tokens/s: 74762.9 | valid tokens/s: 74762.9 | physical tokens/s: 75497.6 | loss tokens/s: 60931.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.656521E-01 | lm-global-supervised-tokens: 1.367344E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.448582E-02 | lm-token-buffer-mib: 2.343132E+02 | global_load_balancing_loss: 9.984642E-01 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11150.743 | params rms: 0.060378 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:56:40.226910 | finish at 2026-10-17 19:31:01 + [2026-10-16 22:38:04] iteration 2049/ 2384 | consumed samples: 65568 | elapsed time per iteration (ms): 223260.8 | throughput per GPU (TFLOP/s/GPU): 25.8 | MFU 8.25% | FLOPs estimator: workload-aware | logical tokens/s: 75146.2 | valid tokens/s: 75146.2 | physical tokens/s: 76016.9 | loss tokens/s: 53176.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.748905E-01 | lm-global-supervised-tokens: 1.187221E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.350729E-02 | lm-token-buffer-mib: 2.331474E+02 | global_load_balancing_loss: 1.000283E+00 | loss scale: 1.0 | grad norm: 0.252 | params norm: 11150.712 | params rms: 0.060377 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:46:32.358617 | finish at 2026-10-17 19:24:37 + [2026-10-16 22:42:06] iteration 2050/ 2384 | consumed samples: 65600 | elapsed time per iteration (ms): 241650.9 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.51% | FLOPs estimator: workload-aware | logical tokens/s: 69427.5 | valid tokens/s: 69427.5 | physical tokens/s: 69697.4 | loss tokens/s: 52111.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.807343E-01 | lm-global-supervised-tokens: 1.259275E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.961928E-02 | lm-token-buffer-mib: 2.331634E+02 | global_load_balancing_loss: 9.980163E-01 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11150.677 | params rms: 0.060377 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:25:11.396342 | finish at 2026-10-17 21:07:18 + [2026-10-16 22:45:53] iteration 2051/ 2384 | consumed samples: 65632 | elapsed time per iteration (ms): 227211.3 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.09% | FLOPs estimator: workload-aware | logical tokens/s: 73839.6 | valid tokens/s: 73839.6 | physical tokens/s: 74735.4 | loss tokens/s: 53367.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.602360E-01 | lm-global-supervised-tokens: 1.212573E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.910051E-02 | lm-token-buffer-mib: 2.373548E+02 | global_load_balancing_loss: 1.001676E+00 | loss scale: 1.0 | grad norm: 0.106 | params norm: 11150.642 | params rms: 0.060377 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:01:01.365724 | finish at 2026-10-17 19:46:55 + [2026-10-16 22:49:38] iteration 2052/ 2384 | consumed samples: 65664 | elapsed time per iteration (ms): 224338.2 | throughput per GPU (TFLOP/s/GPU): 25.9 | MFU 8.31% | FLOPs estimator: workload-aware | logical tokens/s: 74785.1 | valid tokens/s: 74785.1 | physical tokens/s: 75566.2 | loss tokens/s: 54731.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.250310E-01 | lm-global-supervised-tokens: 1.227830E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.382964E-02 | lm-token-buffer-mib: 2.341517E+02 | global_load_balancing_loss: 1.000450E+00 | loss scale: 1.0 | grad norm: 0.129 | params norm: 11150.609 | params rms: 0.060377 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:41:20.266916 | finish at 2026-10-17 19:30:58 + [2026-10-16 22:53:22] iteration 2053/ 2384 | consumed samples: 65696 | elapsed time per iteration (ms): 224209.0 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.01% | FLOPs estimator: workload-aware | logical tokens/s: 74828.3 | valid tokens/s: 74828.3 | physical tokens/s: 75496.7 | loss tokens/s: 53767.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.166589E-01 | lm-global-supervised-tokens: 1.205520E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.696217E-02 | lm-token-buffer-mib: 2.337049E+02 | global_load_balancing_loss: 1.001004E+00 | loss scale: 1.0 | grad norm: 0.132 | params norm: 11150.583 | params rms: 0.060377 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:36:53.191190 | finish at 2026-10-17 19:30:15 + [2026-10-16 22:57:05] iteration 2054/ 2384 | consumed samples: 65728 | elapsed time per iteration (ms): 223152.3 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.13% | FLOPs estimator: workload-aware | logical tokens/s: 75182.6 | valid tokens/s: 75182.6 | physical tokens/s: 76091.1 | loss tokens/s: 57536.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.411821E-01 | lm-global-supervised-tokens: 1.283946E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.001098E-02 | lm-token-buffer-mib: 2.340642E+02 | global_load_balancing_loss: 1.000234E+00 | loss scale: 1.0 | grad norm: 0.214 | params norm: 11150.558 | params rms: 0.060377 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:27:20.260456 | finish at 2026-10-17 19:24:25 + [2026-10-16 23:00:46] iteration 2055/ 2384 | consumed samples: 65760 | elapsed time per iteration (ms): 221430.8 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.20% | FLOPs estimator: workload-aware | logical tokens/s: 75767.2 | valid tokens/s: 75767.2 | physical tokens/s: 76143.6 | loss tokens/s: 57250.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.969335E-01 | lm-global-supervised-tokens: 1.267710E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.598879E-02 | lm-token-buffer-mib: 2.334420E+02 | global_load_balancing_loss: 9.999035E-01 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11150.531 | params rms: 0.060377 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:14:10.725814 | finish at 2026-10-17 19:14:57 + [2026-10-16 23:04:29] iteration 2056/ 2384 | consumed samples: 65792 | elapsed time per iteration (ms): 222474.2 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.32% | FLOPs estimator: workload-aware | logical tokens/s: 75411.9 | valid tokens/s: 75411.9 | physical tokens/s: 75970.6 | loss tokens/s: 55640.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.420437E-01 | lm-global-supervised-tokens: 1.237846E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.274472E-02 | lm-token-buffer-mib: 2.331600E+02 | global_load_balancing_loss: 9.996421E-01 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11150.504 | params rms: 0.060376 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:16:11.526030 | finish at 2026-10-17 19:20:40 + [2026-10-16 23:08:10] iteration 2057/ 2384 | consumed samples: 65824 | elapsed time per iteration (ms): 221458.0 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.83% | FLOPs estimator: workload-aware | logical tokens/s: 75757.9 | valid tokens/s: 75757.9 | physical tokens/s: 76176.5 | loss tokens/s: 56289.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.655000E-01 | lm-global-supervised-tokens: 1.246574E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.275928E-02 | lm-token-buffer-mib: 2.337400E+02 | global_load_balancing_loss: 1.000434E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11150.474 | params rms: 0.060376 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:06:56.779547 | finish at 2026-10-17 19:15:07 + [2026-10-16 23:11:55] iteration 2058/ 2384 | consumed samples: 65856 | elapsed time per iteration (ms): 224699.0 | throughput per GPU (TFLOP/s/GPU): 24.5 | MFU 7.86% | FLOPs estimator: workload-aware | logical tokens/s: 74665.0 | valid tokens/s: 74665.0 | physical tokens/s: 75743.3 | loss tokens/s: 62918.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.762482E-01 | lm-global-supervised-tokens: 1.413779E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.007199E-02 | lm-token-buffer-mib: 2.370211E+02 | global_load_balancing_loss: 9.997505E-01 | loss scale: 1.0 | grad norm: 0.278 | params norm: 11150.445 | params rms: 0.060376 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:20:51.889740 | finish at 2026-10-17 19:32:47 + [2026-10-16 23:15:38] iteration 2059/ 2384 | consumed samples: 65888 | elapsed time per iteration (ms): 222898.8 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.12% | FLOPs estimator: workload-aware | logical tokens/s: 75268.2 | valid tokens/s: 75268.2 | physical tokens/s: 75670.3 | loss tokens/s: 54676.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.137384E-01 | lm-global-supervised-tokens: 1.218726E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.375900E-02 | lm-token-buffer-mib: 2.328274E+02 | global_load_balancing_loss: 1.000980E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11150.421 | params rms: 0.060376 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:07:22.124689 | finish at 2026-10-17 19:23:00 + [2026-10-16 23:19:40] iteration 2060/ 2384 | consumed samples: 65920 | elapsed time per iteration (ms): 242431.3 | throughput per GPU (TFLOP/s/GPU): 18.7 | MFU 5.99% | FLOPs estimator: workload-aware | logical tokens/s: 69204.0 | valid tokens/s: 69204.0 | physical tokens/s: 69881.9 | loss tokens/s: 50367.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.317966E-01 | lm-global-supervised-tokens: 1.221064E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.285345E-02 | lm-token-buffer-mib: 2.357574E+02 | global_load_balancing_loss: 1.000528E+00 | loss scale: 1.0 | grad norm: 0.333 | params norm: 11150.397 | params rms: 0.060376 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:49:07.741253 | finish at 2026-10-17 21:08:48 + [2026-10-16 23:23:23] iteration 2061/ 2384 | consumed samples: 65952 | elapsed time per iteration (ms): 223062.5 | throughput per GPU (TFLOP/s/GPU): 26.1 | MFU 8.36% | FLOPs estimator: workload-aware | logical tokens/s: 75212.8 | valid tokens/s: 75212.8 | physical tokens/s: 75871.8 | loss tokens/s: 58310.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.032173E-01 | lm-global-supervised-tokens: 1.300684E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.012975E-02 | lm-token-buffer-mib: 2.347470E+02 | global_load_balancing_loss: 9.990149E-01 | loss scale: 1.0 | grad norm: 0.102 | params norm: 11150.372 | params rms: 0.060376 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:00:49.191427 | finish at 2026-10-17 19:24:13 + [2026-10-16 23:27:06] iteration 2062/ 2384 | consumed samples: 65984 | elapsed time per iteration (ms): 222355.6 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.95% | FLOPs estimator: workload-aware | logical tokens/s: 75452.0 | valid tokens/s: 75452.0 | physical tokens/s: 75963.6 | loss tokens/s: 58026.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.972634E-01 | lm-global-supervised-tokens: 1.290244E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.324283E-02 | lm-token-buffer-mib: 2.333122E+02 | global_load_balancing_loss: 1.000681E+00 | loss scale: 1.0 | grad norm: 0.135 | params norm: 11150.344 | params rms: 0.060375 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:53:18.510685 | finish at 2026-10-17 19:20:24 + [2026-10-16 23:30:48] iteration 2063/ 2384 | consumed samples: 66016 | elapsed time per iteration (ms): 222304.6 | throughput per GPU (TFLOP/s/GPU): 24.0 | MFU 7.69% | FLOPs estimator: workload-aware | logical tokens/s: 75469.3 | valid tokens/s: 75469.3 | physical tokens/s: 76422.4 | loss tokens/s: 59718.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.415769E-01 | lm-global-supervised-tokens: 1.327578E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.484304E-02 | lm-token-buffer-mib: 2.342094E+02 | global_load_balancing_loss: 1.001551E+00 | loss scale: 1.0 | grad norm: 0.314 | params norm: 11150.314 | params rms: 0.060375 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:49:19.789993 | finish at 2026-10-17 19:20:08 + [2026-10-16 23:34:29] iteration 2064/ 2384 | consumed samples: 66048 | elapsed time per iteration (ms): 220701.2 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.27% | FLOPs estimator: workload-aware | logical tokens/s: 76017.6 | valid tokens/s: 76017.6 | physical tokens/s: 76825.1 | loss tokens/s: 60306.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.351832E-01 | lm-global-supervised-tokens: 1.330966E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.781700E-02 | lm-token-buffer-mib: 2.351714E+02 | global_load_balancing_loss: 9.997619E-01 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11150.285 | params rms: 0.060375 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:37:04.388046 | finish at 2026-10-17 19:11:33 + [2026-10-16 23:38:13] iteration 2065/ 2384 | consumed samples: 66080 | elapsed time per iteration (ms): 224192.2 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.24% | FLOPs estimator: workload-aware | logical tokens/s: 74833.9 | valid tokens/s: 74833.9 | physical tokens/s: 75440.7 | loss tokens/s: 57178.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.724974E-01 | lm-global-supervised-tokens: 1.281904E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.740205E-02 | lm-token-buffer-mib: 2.337397E+02 | global_load_balancing_loss: 1.000484E+00 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11150.254 | params rms: 0.060375 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:51:57.318856 | finish at 2026-10-17 19:30:10 + [2026-10-16 23:41:55] iteration 2066/ 2384 | consumed samples: 66112 | elapsed time per iteration (ms): 222051.1 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.88% | FLOPs estimator: workload-aware | logical tokens/s: 75555.6 | valid tokens/s: 75555.6 | physical tokens/s: 76233.6 | loss tokens/s: 57804.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.545056E-01 | lm-global-supervised-tokens: 1.283556E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.817494E-02 | lm-token-buffer-mib: 2.344322E+02 | global_load_balancing_loss: 1.001682E+00 | loss scale: 1.0 | grad norm: 0.142 | params norm: 11150.222 | params rms: 0.060375 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:36:52.236992 | finish at 2026-10-17 19:18:47 + [2026-10-16 23:45:43] iteration 2067/ 2384 | consumed samples: 66144 | elapsed time per iteration (ms): 228142.6 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.40% | FLOPs estimator: workload-aware | logical tokens/s: 73538.2 | valid tokens/s: 73538.2 | physical tokens/s: 74425.0 | loss tokens/s: 57458.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.185504E-01 | lm-global-supervised-tokens: 1.310872E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.624960E-02 | lm-token-buffer-mib: 2.354453E+02 | global_load_balancing_loss: 1.000994E+00 | loss scale: 1.0 | grad norm: 0.156 | params norm: 11150.191 | params rms: 0.060375 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:05:21.210190 | finish at 2026-10-17 19:51:04 + [2026-10-16 23:49:30] iteration 2068/ 2384 | consumed samples: 66176 | elapsed time per iteration (ms): 226291.1 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.77% | FLOPs estimator: workload-aware | logical tokens/s: 74139.8 | valid tokens/s: 74139.8 | physical tokens/s: 75553.0 | loss tokens/s: 58561.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.816686E-01 | lm-global-supervised-tokens: 1.325186E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.315064E-02 | lm-token-buffer-mib: 2.365169E+02 | global_load_balancing_loss: 1.001976E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11150.161 | params rms: 0.060374 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:51:47.979170 | finish at 2026-10-17 19:41:18 + [2026-10-16 23:53:12] iteration 2069/ 2384 | consumed samples: 66208 | elapsed time per iteration (ms): 222852.5 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.70% | FLOPs estimator: workload-aware | logical tokens/s: 75283.9 | valid tokens/s: 75283.9 | physical tokens/s: 75841.1 | loss tokens/s: 57727.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.188166E-01 | lm-global-supervised-tokens: 1.286467E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.807077E-02 | lm-token-buffer-mib: 2.334690E+02 | global_load_balancing_loss: 1.000499E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11150.130 | params rms: 0.060374 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:29:58.547627 | finish at 2026-10-17 19:23:11 + [2026-10-16 23:57:15] iteration 2070/ 2384 | consumed samples: 66240 | elapsed time per iteration (ms): 242935.6 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.82% | FLOPs estimator: workload-aware | logical tokens/s: 69060.3 | valid tokens/s: 69060.3 | physical tokens/s: 69390.2 | loss tokens/s: 51757.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.310875E-01 | lm-global-supervised-tokens: 1.257371E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.928383E-02 | lm-token-buffer-mib: 2.332070E+02 | global_load_balancing_loss: 9.982639E-01 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11150.098 | params rms: 0.060374 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:11:21.779836 | finish at 2026-10-17 21:08:37 + [2026-10-17 00:00:58] iteration 2071/ 2384 | consumed samples: 66272 | elapsed time per iteration (ms): 222969.4 | throughput per GPU (TFLOP/s/GPU): 26.7 | MFU 8.56% | FLOPs estimator: workload-aware | logical tokens/s: 75244.3 | valid tokens/s: 75244.3 | physical tokens/s: 75619.6 | loss tokens/s: 57317.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.639652E-01 | lm-global-supervised-tokens: 1.278009E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.255852E-02 | lm-token-buffer-mib: 2.328786E+02 | global_load_balancing_loss: 9.999123E-01 | loss scale: 1.0 | grad norm: 0.106 | params norm: 11150.066 | params rms: 0.060374 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:23:09.407850 | finish at 2026-10-17 19:24:08 + [2026-10-17 00:04:42] iteration 2072/ 2384 | consumed samples: 66304 | elapsed time per iteration (ms): 223634.0 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.41% | FLOPs estimator: workload-aware | logical tokens/s: 75020.7 | valid tokens/s: 75020.7 | physical tokens/s: 75742.3 | loss tokens/s: 60570.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.196500E-01 | lm-global-supervised-tokens: 1.354558E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.933094E-02 | lm-token-buffer-mib: 2.343762E+02 | global_load_balancing_loss: 9.992463E-01 | loss scale: 1.0 | grad norm: 0.150 | params norm: 11150.035 | params rms: 0.060374 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:22:53.798647 | finish at 2026-10-17 19:27:36 + [2026-10-17 00:08:26] iteration 2073/ 2384 | consumed samples: 66336 | elapsed time per iteration (ms): 224019.2 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.24% | FLOPs estimator: workload-aware | logical tokens/s: 74891.7 | valid tokens/s: 74891.7 | physical tokens/s: 75332.1 | loss tokens/s: 52562.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.974453E-01 | lm-global-supervised-tokens: 1.177502E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.576853E-02 | lm-token-buffer-mib: 2.332092E+02 | global_load_balancing_loss: 1.000232E+00 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11150.003 | params rms: 0.060374 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:21:09.977752 | finish at 2026-10-17 19:29:36 + [2026-10-17 00:12:10] iteration 2074/ 2384 | consumed samples: 66368 | elapsed time per iteration (ms): 223808.0 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.35% | FLOPs estimator: workload-aware | logical tokens/s: 74962.4 | valid tokens/s: 74962.4 | physical tokens/s: 75774.1 | loss tokens/s: 61314.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.468934E-01 | lm-global-supervised-tokens: 1.372260E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.067517E-02 | lm-token-buffer-mib: 2.359700E+02 | global_load_balancing_loss: 1.000688E+00 | loss scale: 1.0 | grad norm: 0.104 | params norm: 11149.963 | params rms: 0.060373 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:16:20.493479 | finish at 2026-10-17 19:28:30 + [2026-10-17 00:15:50] iteration 2075/ 2384 | consumed samples: 66400 | elapsed time per iteration (ms): 220184.8 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.93% | FLOPs estimator: workload-aware | logical tokens/s: 76195.8 | valid tokens/s: 76195.8 | physical tokens/s: 77198.6 | loss tokens/s: 60081.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.117234E-01 | lm-global-supervised-tokens: 1.322912E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.177086E-02 | lm-token-buffer-mib: 2.344694E+02 | global_load_balancing_loss: 1.001085E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11149.928 | params rms: 0.060373 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:53:57.096468 | finish at 2026-10-17 19:09:47 + [2026-10-17 00:19:30] iteration 2076/ 2384 | consumed samples: 66432 | elapsed time per iteration (ms): 219677.9 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.45% | FLOPs estimator: workload-aware | logical tokens/s: 76371.8 | valid tokens/s: 76371.8 | physical tokens/s: 76896.3 | loss tokens/s: 57768.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.735673E-01 | lm-global-supervised-tokens: 1.269039E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.696281E-02 | lm-token-buffer-mib: 2.336260E+02 | global_load_balancing_loss: 1.001694E+00 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11149.896 | params rms: 0.060373 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:47:40.791534 | finish at 2026-10-17 19:07:10 + [2026-10-17 00:23:12] iteration 2077/ 2384 | consumed samples: 66464 | elapsed time per iteration (ms): 222060.6 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.97% | FLOPs estimator: workload-aware | logical tokens/s: 75552.4 | valid tokens/s: 75552.4 | physical tokens/s: 76421.4 | loss tokens/s: 60139.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.557148E-01 | lm-global-supervised-tokens: 1.335468E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.843280E-02 | lm-token-buffer-mib: 2.340334E+02 | global_load_balancing_loss: 1.001496E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11149.865 | params rms: 0.060373 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:56:12.599529 | finish at 2026-10-17 19:19:24 + [2026-10-17 00:26:54] iteration 2078/ 2384 | consumed samples: 66496 | elapsed time per iteration (ms): 222432.3 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.20% | FLOPs estimator: workload-aware | logical tokens/s: 75425.9 | valid tokens/s: 75425.9 | physical tokens/s: 76161.3 | loss tokens/s: 59534.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.272400E-01 | lm-global-supervised-tokens: 1.324243E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.185866E-02 | lm-token-buffer-mib: 2.331470E+02 | global_load_balancing_loss: 1.000704E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11149.831 | params rms: 0.060373 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:54:24.295574 | finish at 2026-10-17 19:21:18 + [2026-10-17 00:30:35] iteration 2079/ 2384 | consumed samples: 66528 | elapsed time per iteration (ms): 220551.5 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.56% | FLOPs estimator: workload-aware | logical tokens/s: 76069.3 | valid tokens/s: 76069.3 | physical tokens/s: 77829.6 | loss tokens/s: 55601.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.880089E-01 | lm-global-supervised-tokens: 1.226299E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.745198E-02 | lm-token-buffer-mib: 2.343372E+02 | global_load_balancing_loss: 1.001284E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11149.798 | params rms: 0.060373 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:41:08.203089 | finish at 2026-10-17 19:11:43 + [2026-10-17 00:34:42] iteration 2080/ 2384 | consumed samples: 66560 | elapsed time per iteration (ms): 247092.9 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.40% | FLOPs estimator: workload-aware | logical tokens/s: 67898.3 | valid tokens/s: 67898.3 | physical tokens/s: 68504.0 | loss tokens/s: 51939.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.825091E-01 | lm-global-supervised-tokens: 1.283378E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.350239E-02 | lm-token-buffer-mib: 2.343970E+02 | global_load_balancing_loss: 9.992663E-01 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11149.769 | params rms: 0.060372 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:51:56.237335 | finish at 2026-10-17 21:26:38 +times across ranks (ms): + save-checkpoint: + rank 0: 24235.24 + rank 1: 24235.23 + rank 2: 24235.28 + rank 3: 24235.24 + rank 4: 24235.24 + rank 5: 24235.25 + rank 6: 24235.26 + rank 7: 24235.27 + rank 8: 24233.71 + rank 9: 24233.70 + rank 10: 24233.72 + rank 11: 24233.71 + rank 12: 24233.68 + rank 13: 24233.71 + rank 14: 24233.68 + rank 15: 24233.73 + rank 16: 24233.71 + rank 17: 24233.70 + rank 18: 24233.69 + rank 19: 24233.67 + rank 20: 24233.68 + rank 21: 24233.68 + rank 22: 24233.69 + rank 23: 24233.69 + rank 24: 24233.77 + rank 25: 24233.78 + rank 26: 24233.94 + rank 27: 24233.78 + rank 28: 24233.80 + rank 29: 24233.80 + rank 30: 24233.78 + rank 31: 24233.79 + rank 32: 24233.67 + rank 33: 24233.68 + rank 34: 24233.68 + rank 35: 24233.69 + rank 36: 24233.64 + rank 37: 24233.67 + rank 38: 24233.65 + rank 39: 24233.68 + rank 40: 24233.58 + rank 41: 24233.55 + rank 42: 24233.54 + rank 43: 24233.58 + rank 44: 24233.61 + rank 45: 24233.55 + rank 46: 24233.54 + rank 47: 24233.58 + rank 48: 24233.41 + rank 49: 24233.44 + rank 50: 24233.45 + rank 51: 24233.38 + rank 52: 24233.40 + rank 53: 24233.45 + rank 54: 24233.39 + rank 55: 24233.39 + rank 56: 24233.82 + rank 57: 24233.88 + rank 58: 24233.82 + rank 59: 24233.82 + rank 60: 24233.82 + rank 61: 24233.80 + rank 62: 24233.81 + rank 63: 24233.82 + [2026-10-17 00:38:59] iteration 2081/ 2384 | consumed samples: 66592 | elapsed time per iteration (ms): 232384.7 | throughput per GPU (TFLOP/s/GPU): 24.9 | MFU 7.99% | FLOPs estimator: workload-aware | logical tokens/s: 72195.7 | valid tokens/s: 72195.7 | physical tokens/s: 72840.6 | loss tokens/s: 55846.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.609133E-01 | lm-global-supervised-tokens: 1.297789E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.353585E+00 | lm-token-buffer-mib: 2.355309E+02 | global_load_balancing_loss: 9.992558E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11149.744 | params rms: 0.060372 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:33:32.559350 | finish at 2026-10-17 20:12:31 + [2026-10-17 00:42:44] iteration 2082/ 2384 | consumed samples: 66624 | elapsed time per iteration (ms): 225800.6 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.80% | FLOPs estimator: workload-aware | logical tokens/s: 74300.9 | valid tokens/s: 74300.9 | physical tokens/s: 74870.2 | loss tokens/s: 56722.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.730042E-01 | lm-global-supervised-tokens: 1.280804E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.296534E-02 | lm-token-buffer-mib: 2.336937E+02 | global_load_balancing_loss: 1.000124E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11149.716 | params rms: 0.060372 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:56:31.776896 | finish at 2026-10-17 19:39:16 + [2026-10-17 00:46:30] iteration 2083/ 2384 | consumed samples: 66656 | elapsed time per iteration (ms): 225118.7 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.06% | FLOPs estimator: workload-aware | logical tokens/s: 74525.9 | valid tokens/s: 74525.9 | physical tokens/s: 76068.9 | loss tokens/s: 52772.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.400972E-01 | lm-global-supervised-tokens: 1.188006E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.459452E-02 | lm-token-buffer-mib: 2.382769E+02 | global_load_balancing_loss: 1.001501E+00 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11149.689 | params rms: 0.060372 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:49:20.737679 | finish at 2026-10-17 19:35:50 + [2026-10-17 00:50:13] iteration 2084/ 2384 | consumed samples: 66688 | elapsed time per iteration (ms): 223011.3 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.77% | FLOPs estimator: workload-aware | logical tokens/s: 75230.3 | valid tokens/s: 75230.3 | physical tokens/s: 76046.5 | loss tokens/s: 57171.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.324401E-01 | lm-global-supervised-tokens: 1.274977E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.056557E-02 | lm-token-buffer-mib: 2.337904E+02 | global_load_balancing_loss: 1.000240E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11149.660 | params rms: 0.060372 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:35:03.397322 | finish at 2026-10-17 19:25:16 + [2026-10-17 00:53:58] iteration 2085/ 2384 | consumed samples: 66720 | elapsed time per iteration (ms): 225517.2 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.43% | FLOPs estimator: workload-aware | logical tokens/s: 74394.3 | valid tokens/s: 74394.3 | physical tokens/s: 75878.4 | loss tokens/s: 58112.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.593706E-01 | lm-global-supervised-tokens: 1.310531E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.346146E-02 | lm-token-buffer-mib: 2.373569E+02 | global_load_balancing_loss: 9.996004E-01 | loss scale: 1.0 | grad norm: 0.157 | params norm: 11149.633 | params rms: 0.060372 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:43:49.633531 | finish at 2026-10-17 19:37:48 + [2026-10-17 00:57:43] iteration 2086/ 2384 | consumed samples: 66752 | elapsed time per iteration (ms): 224604.1 | throughput per GPU (TFLOP/s/GPU): 19.1 | MFU 6.13% | FLOPs estimator: workload-aware | logical tokens/s: 74696.8 | valid tokens/s: 74696.8 | physical tokens/s: 75969.4 | loss tokens/s: 58471.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.586580E-01 | lm-global-supervised-tokens: 1.313291E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.005592E-02 | lm-token-buffer-mib: 2.374013E+02 | global_load_balancing_loss: 1.000106E+00 | loss scale: 1.0 | grad norm: 0.110 | params norm: 11149.607 | params rms: 0.060371 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:35:32.024212 | finish at 2026-10-17 19:33:15 + [2026-10-17 01:01:23] iteration 2087/ 2384 | consumed samples: 66784 | elapsed time per iteration (ms): 220218.9 | throughput per GPU (TFLOP/s/GPU): 24.2 | MFU 7.77% | FLOPs estimator: workload-aware | logical tokens/s: 76184.1 | valid tokens/s: 76184.1 | physical tokens/s: 76827.1 | loss tokens/s: 58909.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.529522E-01 | lm-global-supervised-tokens: 1.297295E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.504236E-02 | lm-token-buffer-mib: 2.342586E+02 | global_load_balancing_loss: 1.000902E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11149.581 | params rms: 0.060371 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:10:05.005925 | finish at 2026-10-17 19:11:28 + [2026-10-17 01:05:07] iteration 2088/ 2384 | consumed samples: 66816 | elapsed time per iteration (ms): 223916.3 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.55% | FLOPs estimator: workload-aware | logical tokens/s: 74926.2 | valid tokens/s: 74926.2 | physical tokens/s: 75594.0 | loss tokens/s: 56362.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.926745E-01 | lm-global-supervised-tokens: 1.262044E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.255804E-02 | lm-token-buffer-mib: 2.326982E+02 | global_load_balancing_loss: 1.000064E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11149.553 | params rms: 0.060371 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:24:39.219807 | finish at 2026-10-17 19:29:46 + [2026-10-17 01:08:49] iteration 2089/ 2384 | consumed samples: 66848 | elapsed time per iteration (ms): 222129.9 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.70% | FLOPs estimator: workload-aware | logical tokens/s: 75528.9 | valid tokens/s: 75528.9 | physical tokens/s: 75978.9 | loss tokens/s: 56324.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.360655E-01 | lm-global-supervised-tokens: 1.251138E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.365280E-02 | lm-token-buffer-mib: 2.328980E+02 | global_load_balancing_loss: 1.000445E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11149.521 | params rms: 0.060371 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:12:08.320705 | finish at 2026-10-17 19:20:57 + [2026-10-17 01:12:54] iteration 2090/ 2384 | consumed samples: 66880 | elapsed time per iteration (ms): 245118.7 | throughput per GPU (TFLOP/s/GPU): 17.6 | MFU 5.63% | FLOPs estimator: workload-aware | logical tokens/s: 68445.2 | valid tokens/s: 68445.2 | physical tokens/s: 69292.8 | loss tokens/s: 51473.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.584202E-01 | lm-global-supervised-tokens: 1.261714E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.454566E-02 | lm-token-buffer-mib: 2.344917E+02 | global_load_balancing_loss: 9.996715E-01 | loss scale: 1.0 | grad norm: 0.124 | params norm: 11149.488 | params rms: 0.060371 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:01:04.891850 | finish at 2026-10-17 21:13:59 + [2026-10-17 01:16:38] iteration 2091/ 2384 | consumed samples: 66912 | elapsed time per iteration (ms): 223800.1 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.61% | FLOPs estimator: workload-aware | logical tokens/s: 74965.1 | valid tokens/s: 74965.1 | physical tokens/s: 75924.0 | loss tokens/s: 55262.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.034779E-01 | lm-global-supervised-tokens: 1.236777E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.848736E-02 | lm-token-buffer-mib: 2.361853E+02 | global_load_balancing_loss: 1.000998E+00 | loss scale: 1.0 | grad norm: 0.124 | params norm: 11149.459 | params rms: 0.060371 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:12:53.433867 | finish at 2026-10-17 19:29:31 + [2026-10-17 01:20:22] iteration 2092/ 2384 | consumed samples: 66944 | elapsed time per iteration (ms): 224386.4 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.64% | FLOPs estimator: workload-aware | logical tokens/s: 74769.2 | valid tokens/s: 74769.2 | physical tokens/s: 75318.4 | loss tokens/s: 56547.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.963782E-01 | lm-global-supervised-tokens: 1.268855E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.251424E-02 | lm-token-buffer-mib: 2.325289E+02 | global_load_balancing_loss: 1.000410E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11149.427 | params rms: 0.060371 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:12:00.823504 | finish at 2026-10-17 19:32:23 + [2026-10-17 01:24:03] iteration 2093/ 2384 | consumed samples: 66976 | elapsed time per iteration (ms): 220739.7 | throughput per GPU (TFLOP/s/GPU): 18.6 | MFU 5.95% | FLOPs estimator: workload-aware | logical tokens/s: 76004.5 | valid tokens/s: 76004.5 | physical tokens/s: 76857.8 | loss tokens/s: 57018.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.580691E-01 | lm-global-supervised-tokens: 1.258626E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.695273E-02 | lm-token-buffer-mib: 2.331150E+02 | global_load_balancing_loss: 1.000758E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11149.393 | params rms: 0.060370 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:50:35.244259 | finish at 2026-10-17 19:14:38 + [2026-10-17 01:27:45] iteration 2094/ 2384 | consumed samples: 67008 | elapsed time per iteration (ms): 221825.8 | throughput per GPU (TFLOP/s/GPU): 18.8 | MFU 6.03% | FLOPs estimator: workload-aware | logical tokens/s: 75632.4 | valid tokens/s: 75632.4 | physical tokens/s: 76551.0 | loss tokens/s: 56842.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.035618E-01 | lm-global-supervised-tokens: 1.260917E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.436553E-02 | lm-token-buffer-mib: 2.368176E+02 | global_load_balancing_loss: 9.998862E-01 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11149.359 | params rms: 0.060370 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:52:09.491041 | finish at 2026-10-17 19:19:54 + [2026-10-17 01:31:26] iteration 2095/ 2384 | consumed samples: 67040 | elapsed time per iteration (ms): 220920.1 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.34% | FLOPs estimator: workload-aware | logical tokens/s: 75942.4 | valid tokens/s: 75942.4 | physical tokens/s: 76647.6 | loss tokens/s: 55653.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.904767E-01 | lm-global-supervised-tokens: 1.229488E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.647182E-02 | lm-token-buffer-mib: 2.330202E+02 | global_load_balancing_loss: 1.001041E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11149.324 | params rms: 0.060370 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:44:05.900004 | finish at 2026-10-17 19:15:32 + [2026-10-17 01:35:10] iteration 2096/ 2384 | consumed samples: 67072 | elapsed time per iteration (ms): 223995.6 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.93% | FLOPs estimator: workload-aware | logical tokens/s: 74899.7 | valid tokens/s: 74899.7 | physical tokens/s: 75732.9 | loss tokens/s: 57363.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.993503E-01 | lm-global-supervised-tokens: 1.284921E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.755107E-02 | lm-token-buffer-mib: 2.342822E+02 | global_load_balancing_loss: 1.001043E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11149.288 | params rms: 0.060370 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:55:10.743027 | finish at 2026-10-17 19:30:20 + [2026-10-17 01:38:52] iteration 2097/ 2384 | consumed samples: 67104 | elapsed time per iteration (ms): 222153.6 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.33% | FLOPs estimator: workload-aware | logical tokens/s: 75520.4 | valid tokens/s: 75520.4 | physical tokens/s: 76046.0 | loss tokens/s: 55296.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.707108E-01 | lm-global-supervised-tokens: 1.228426E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.251418E-02 | lm-token-buffer-mib: 2.350306E+02 | global_load_balancing_loss: 1.000056E+00 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11149.256 | params rms: 0.060370 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:42:38.075872 | finish at 2026-10-17 19:21:30 + [2026-10-17 01:42:35] iteration 2098/ 2384 | consumed samples: 67136 | elapsed time per iteration (ms): 222946.3 | throughput per GPU (TFLOP/s/GPU): 19.5 | MFU 6.24% | FLOPs estimator: workload-aware | logical tokens/s: 75252.2 | valid tokens/s: 75252.2 | physical tokens/s: 76009.9 | loss tokens/s: 56967.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.005484E-01 | lm-global-supervised-tokens: 1.270060E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.421147E-02 | lm-token-buffer-mib: 2.346653E+02 | global_load_balancing_loss: 1.000294E+00 | loss scale: 1.0 | grad norm: 0.117 | params norm: 11149.225 | params rms: 0.060369 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:42:42.628921 | finish at 2026-10-17 19:25:17 + [2026-10-17 01:46:19] iteration 2099/ 2384 | consumed samples: 67168 | elapsed time per iteration (ms): 223957.3 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.71% | FLOPs estimator: workload-aware | logical tokens/s: 74912.5 | valid tokens/s: 74912.5 | physical tokens/s: 76101.1 | loss tokens/s: 58413.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.837449E-01 | lm-global-supervised-tokens: 1.308216E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.971628E-02 | lm-token-buffer-mib: 2.372285E+02 | global_load_balancing_loss: 9.989343E-01 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11149.195 | params rms: 0.060369 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:43:47.825797 | finish at 2026-10-17 19:30:07 + [2026-10-17 01:50:25] iteration 2100/ 2384 | consumed samples: 67200 | elapsed time per iteration (ms): 246627.5 | throughput per GPU (TFLOP/s/GPU): 19.6 | MFU 6.27% | FLOPs estimator: workload-aware | logical tokens/s: 68026.3 | valid tokens/s: 68026.3 | physical tokens/s: 69013.4 | loss tokens/s: 53933.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.904126E-01 | lm-global-supervised-tokens: 1.330142E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.915448E-02 | lm-token-buffer-mib: 2.343763E+02 | global_load_balancing_loss: 1.000191E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11149.164 | params rms: 0.060369 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:27:22.205818 | finish at 2026-10-17 21:17:48 + [2026-10-17 01:54:06] iteration 2101/ 2384 | consumed samples: 67232 | elapsed time per iteration (ms): 220655.4 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.11% | FLOPs estimator: workload-aware | logical tokens/s: 76033.4 | valid tokens/s: 76033.4 | physical tokens/s: 76899.1 | loss tokens/s: 58547.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.840093E-01 | lm-global-supervised-tokens: 1.291883E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.264868E-02 | lm-token-buffer-mib: 2.336571E+02 | global_load_balancing_loss: 9.995576E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11149.135 | params rms: 0.060369 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:20:45.464716 | finish at 2026-10-17 19:14:52 + [2026-10-17 01:57:55] iteration 2102/ 2384 | consumed samples: 67264 | elapsed time per iteration (ms): 228648.8 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.57% | FLOPs estimator: workload-aware | logical tokens/s: 73375.4 | valid tokens/s: 73375.4 | physical tokens/s: 74489.5 | loss tokens/s: 53943.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.625893E-01 | lm-global-supervised-tokens: 1.233420E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.186025E-02 | lm-token-buffer-mib: 2.349768E+02 | global_load_balancing_loss: 1.000678E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11149.104 | params rms: 0.060369 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:54:38.974480 | finish at 2026-10-17 19:52:34 + [2026-10-17 02:01:38] iteration 2103/ 2384 | consumed samples: 67296 | elapsed time per iteration (ms): 223335.2 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.47% | FLOPs estimator: workload-aware | logical tokens/s: 75121.1 | valid tokens/s: 75121.1 | physical tokens/s: 75538.5 | loss tokens/s: 56458.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.744891E-01 | lm-global-supervised-tokens: 1.260927E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.075890E-02 | lm-token-buffer-mib: 2.337472E+02 | global_load_balancing_loss: 9.995740E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11149.073 | params rms: 0.060369 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:25:57.201202 | finish at 2026-10-17 19:27:35 + [2026-10-17 02:05:20] iteration 2104/ 2384 | consumed samples: 67328 | elapsed time per iteration (ms): 221707.8 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.09% | FLOPs estimator: workload-aware | logical tokens/s: 75672.6 | valid tokens/s: 75672.5 | physical tokens/s: 76148.4 | loss tokens/s: 57632.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.362641E-01 | lm-global-supervised-tokens: 1.277756E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.510568E-02 | lm-token-buffer-mib: 2.341996E+02 | global_load_balancing_loss: 1.000794E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11149.042 | params rms: 0.060368 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:14:38.178034 | finish at 2026-10-17 19:19:58 + [2026-10-17 02:09:02] iteration 2105/ 2384 | consumed samples: 67360 | elapsed time per iteration (ms): 222561.4 | throughput per GPU (TFLOP/s/GPU): 20.2 | MFU 6.46% | FLOPs estimator: workload-aware | logical tokens/s: 75382.3 | valid tokens/s: 75382.3 | physical tokens/s: 76466.2 | loss tokens/s: 56441.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.914627E-01 | lm-global-supervised-tokens: 1.256160E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.655443E-02 | lm-token-buffer-mib: 2.350759E+02 | global_load_balancing_loss: 1.000977E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11149.011 | params rms: 0.060368 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:14:54.632511 | finish at 2026-10-17 19:23:57 + [2026-10-17 02:12:47] iteration 2106/ 2384 | consumed samples: 67392 | elapsed time per iteration (ms): 224614.7 | throughput per GPU (TFLOP/s/GPU): 24.6 | MFU 7.89% | FLOPs estimator: workload-aware | logical tokens/s: 74693.3 | valid tokens/s: 74693.3 | physical tokens/s: 75227.0 | loss tokens/s: 58805.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.776562E-01 | lm-global-supervised-tokens: 1.320866E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.632849E-02 | lm-token-buffer-mib: 2.332365E+02 | global_load_balancing_loss: 1.001506E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11148.980 | params rms: 0.060368 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:20:42.889074 | finish at 2026-10-17 19:33:30 + [2026-10-17 02:16:29] iteration 2107/ 2384 | consumed samples: 67424 | elapsed time per iteration (ms): 222217.7 | throughput per GPU (TFLOP/s/GPU): 20.2 | MFU 6.49% | FLOPs estimator: workload-aware | logical tokens/s: 75498.8 | valid tokens/s: 75498.8 | physical tokens/s: 76199.1 | loss tokens/s: 57310.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.959680E-01 | lm-global-supervised-tokens: 1.273546E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.283659E-02 | lm-token-buffer-mib: 2.332350E+02 | global_load_balancing_loss: 1.001347E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11148.950 | params rms: 0.060368 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:05:54.303496 | finish at 2026-10-17 19:22:23 + [2026-10-17 02:20:13] iteration 2108/ 2384 | consumed samples: 67456 | elapsed time per iteration (ms): 223684.6 | throughput per GPU (TFLOP/s/GPU): 24.1 | MFU 7.73% | FLOPs estimator: workload-aware | logical tokens/s: 75003.8 | valid tokens/s: 75003.8 | physical tokens/s: 75506.9 | loss tokens/s: 55891.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.150554E-01 | lm-global-supervised-tokens: 1.250204E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.590570E-02 | lm-token-buffer-mib: 2.346701E+02 | global_load_balancing_loss: 9.999435E-01 | loss scale: 1.0 | grad norm: 0.031 | params norm: 11148.921 | params rms: 0.060368 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:08:56.948381 | finish at 2026-10-17 19:29:10 + [2026-10-17 02:23:57] iteration 2109/ 2384 | consumed samples: 67488 | elapsed time per iteration (ms): 223652.5 | throughput per GPU (TFLOP/s/GPU): 25.8 | MFU 8.28% | FLOPs estimator: workload-aware | logical tokens/s: 75014.7 | valid tokens/s: 75014.7 | physical tokens/s: 75437.6 | loss tokens/s: 57958.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.288077E-01 | lm-global-supervised-tokens: 1.296253E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.195869E-02 | lm-token-buffer-mib: 2.334222E+02 | global_load_balancing_loss: 1.000178E+00 | loss scale: 1.0 | grad norm: 0.103 | params norm: 11148.889 | params rms: 0.060368 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:05:04.441082 | finish at 2026-10-17 19:29:01 + [2026-10-17 02:28:02] iteration 2110/ 2384 | consumed samples: 67520 | elapsed time per iteration (ms): 245639.4 | throughput per GPU (TFLOP/s/GPU): 17.8 | MFU 5.72% | FLOPs estimator: workload-aware | logical tokens/s: 68300.1 | valid tokens/s: 68300.1 | physical tokens/s: 69198.0 | loss tokens/s: 54319.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.440075E-01 | lm-global-supervised-tokens: 1.334310E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.490736E-02 | lm-token-buffer-mib: 2.340833E+02 | global_load_balancing_loss: 9.997463E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11148.852 | params rms: 0.060367 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:41:45.207818 | finish at 2026-10-17 21:09:47 + [2026-10-17 02:31:47] iteration 2111/ 2384 | consumed samples: 67552 | elapsed time per iteration (ms): 225046.6 | throughput per GPU (TFLOP/s/GPU): 25.4 | MFU 8.15% | FLOPs estimator: workload-aware | logical tokens/s: 74549.8 | valid tokens/s: 74549.8 | physical tokens/s: 74857.4 | loss tokens/s: 54308.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.758167E-01 | lm-global-supervised-tokens: 1.222186E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.624123E-02 | lm-token-buffer-mib: 2.329364E+02 | global_load_balancing_loss: 9.993583E-01 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11148.817 | params rms: 0.060367 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:03:57.718899 | finish at 2026-10-17 19:35:45 + [2026-10-17 02:35:30] iteration 2112/ 2384 | consumed samples: 67584 | elapsed time per iteration (ms): 222521.6 | throughput per GPU (TFLOP/s/GPU): 24.3 | MFU 7.80% | FLOPs estimator: workload-aware | logical tokens/s: 75395.8 | valid tokens/s: 75395.8 | physical tokens/s: 76183.4 | loss tokens/s: 60537.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.518953E-01 | lm-global-supervised-tokens: 1.347079E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.390938E-02 | lm-token-buffer-mib: 2.329159E+02 | global_load_balancing_loss: 9.989558E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11148.786 | params rms: 0.060367 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:48:45.873581 | finish at 2026-10-17 19:24:16 + [2026-10-17 02:39:15] iteration 2113/ 2384 | consumed samples: 67616 | elapsed time per iteration (ms): 225286.7 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.55% | FLOPs estimator: workload-aware | logical tokens/s: 74470.4 | valid tokens/s: 74470.4 | physical tokens/s: 74992.7 | loss tokens/s: 56644.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.404223E-01 | lm-global-supervised-tokens: 1.276127E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.640540E-02 | lm-token-buffer-mib: 2.331731E+02 | global_load_balancing_loss: 1.001130E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11148.760 | params rms: 0.060367 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:57:32.686076 | finish at 2026-10-17 19:36:48 + [2026-10-17 02:42:55] iteration 2114/ 2384 | consumed samples: 67648 | elapsed time per iteration (ms): 219656.1 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.40% | FLOPs estimator: workload-aware | logical tokens/s: 76379.4 | valid tokens/s: 76379.4 | physical tokens/s: 77889.9 | loss tokens/s: 57327.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.657475E-01 | lm-global-supervised-tokens: 1.259225E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.827881E-02 | lm-token-buffer-mib: 2.340884E+02 | global_load_balancing_loss: 1.000376E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11148.738 | params rms: 0.060367 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:28:27.151580 | finish at 2026-10-17 19:11:22 + [2026-10-17 02:46:38] iteration 2115/ 2384 | consumed samples: 67680 | elapsed time per iteration (ms): 222923.4 | throughput per GPU (TFLOP/s/GPU): 19.9 | MFU 6.36% | FLOPs estimator: workload-aware | logical tokens/s: 75260.0 | valid tokens/s: 75260.0 | physical tokens/s: 76146.8 | loss tokens/s: 59589.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.606621E-01 | lm-global-supervised-tokens: 1.328386E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.150596E-02 | lm-token-buffer-mib: 2.351248E+02 | global_load_balancing_loss: 9.996673E-01 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11148.716 | params rms: 0.060367 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:39:26.387333 | finish at 2026-10-17 19:26:04 + [2026-10-17 02:50:20] iteration 2116/ 2384 | consumed samples: 67712 | elapsed time per iteration (ms): 221938.2 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.06% | FLOPs estimator: workload-aware | logical tokens/s: 75593.9 | valid tokens/s: 75593.9 | physical tokens/s: 76366.6 | loss tokens/s: 59715.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.229554E-01 | lm-global-supervised-tokens: 1.325317E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.280313E-02 | lm-token-buffer-mib: 2.346736E+02 | global_load_balancing_loss: 1.000936E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11148.691 | params rms: 0.060367 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:31:19.427823 | finish at 2026-10-17 19:21:39 + [2026-10-17 02:54:01] iteration 2117/ 2384 | consumed samples: 67744 | elapsed time per iteration (ms): 221422.8 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 7.00% | FLOPs estimator: workload-aware | logical tokens/s: 75770.0 | valid tokens/s: 75770.0 | physical tokens/s: 76434.6 | loss tokens/s: 59452.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.108285E-01 | lm-global-supervised-tokens: 1.316412E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.387864E-02 | lm-token-buffer-mib: 2.339328E+02 | global_load_balancing_loss: 1.001748E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11148.668 | params rms: 0.060366 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:25:19.886790 | finish at 2026-10-17 19:19:21 + [2026-10-17 02:57:42] iteration 2118/ 2384 | consumed samples: 67776 | elapsed time per iteration (ms): 221420.3 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.66% | FLOPs estimator: workload-aware | logical tokens/s: 75770.8 | valid tokens/s: 75770.8 | physical tokens/s: 77042.1 | loss tokens/s: 58243.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.969320E-01 | lm-global-supervised-tokens: 1.289621E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.346905E-02 | lm-token-buffer-mib: 2.351046E+02 | global_load_balancing_loss: 1.001365E+00 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11148.647 | params rms: 0.060366 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:21:37.811408 | finish at 2026-10-17 19:19:20 + [2026-10-17 03:01:26] iteration 2119/ 2384 | consumed samples: 67808 | elapsed time per iteration (ms): 224039.5 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.72% | FLOPs estimator: workload-aware | logical tokens/s: 74885.0 | valid tokens/s: 74885.0 | physical tokens/s: 76278.3 | loss tokens/s: 55342.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.415414E-01 | lm-global-supervised-tokens: 1.239884E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.719884E-02 | lm-token-buffer-mib: 2.344816E+02 | global_load_balancing_loss: 1.001046E+00 | loss scale: 1.0 | grad norm: 0.238 | params norm: 11148.628 | params rms: 0.060366 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:29:30.476029 | finish at 2026-10-17 19:30:57 + [2026-10-17 03:05:34] iteration 2120/ 2384 | consumed samples: 67840 | elapsed time per iteration (ms): 248067.6 | throughput per GPU (TFLOP/s/GPU): 18.4 | MFU 5.90% | FLOPs estimator: workload-aware | logical tokens/s: 67631.6 | valid tokens/s: 67631.6 | physical tokens/s: 68452.1 | loss tokens/s: 51489.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.320377E-01 | lm-global-supervised-tokens: 1.277296E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.203087E-02 | lm-token-buffer-mib: 2.367964E+02 | global_load_balancing_loss: 1.000798E+00 | loss scale: 1.0 | grad norm: 0.342 | params norm: 11148.607 | params rms: 0.060366 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:11:29.834255 | finish at 2026-10-17 21:17:04 +times across ranks (ms): + save-checkpoint: + rank 0: 24330.55 + rank 1: 24330.57 + rank 2: 24330.58 + rank 3: 24330.62 + rank 4: 24330.57 + rank 5: 24330.59 + rank 6: 24330.56 + rank 7: 24330.56 + rank 8: 24331.49 + rank 9: 24331.53 + rank 10: 24331.57 + rank 11: 24331.49 + rank 12: 24331.52 + rank 13: 24331.49 + rank 14: 24331.51 + rank 15: 24331.48 + rank 16: 24330.45 + rank 17: 24330.47 + rank 18: 24330.45 + rank 19: 24330.45 + rank 20: 24330.45 + rank 21: 24330.47 + rank 22: 24330.49 + rank 23: 24330.51 + rank 24: 24330.42 + rank 25: 24330.43 + rank 26: 24330.42 + rank 27: 24330.44 + rank 28: 24330.40 + rank 29: 24330.47 + rank 30: 24330.43 + rank 31: 24330.42 + rank 32: 24330.37 + rank 33: 24330.38 + rank 34: 24330.36 + rank 35: 24330.39 + rank 36: 24330.33 + rank 37: 24330.36 + rank 38: 24330.32 + rank 39: 24330.34 + rank 40: 24331.74 + rank 41: 24331.74 + rank 42: 24331.73 + rank 43: 24331.79 + rank 44: 24331.74 + rank 45: 24331.74 + rank 46: 24331.73 + rank 47: 24331.73 + rank 48: 24330.69 + rank 49: 24330.71 + rank 50: 24330.71 + rank 51: 24331.12 + rank 52: 24330.67 + rank 53: 24330.71 + rank 54: 24330.69 + rank 55: 24330.68 + rank 56: 24330.57 + rank 57: 24330.56 + rank 58: 24330.58 + rank 59: 24330.55 + rank 60: 24330.60 + rank 61: 24330.55 + rank 62: 24330.55 + rank 63: 24330.57 + [2026-10-17 03:09:35] iteration 2121/ 2384 | consumed samples: 67872 | elapsed time per iteration (ms): 216572.7 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.57% | FLOPs estimator: workload-aware | logical tokens/s: 77466.7 | valid tokens/s: 77466.7 | physical tokens/s: 78061.2 | loss tokens/s: 57576.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.749724E-01 | lm-global-supervised-tokens: 1.246952E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.380630E-02 | lm-token-buffer-mib: 2.327495E+02 | global_load_balancing_loss: 9.998278E-01 | loss scale: 1.0 | grad norm: 0.161 | params norm: 11148.586 | params rms: 0.060366 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:49:18.628195 | finish at 2026-10-17 18:58:54 + [2026-10-17 03:13:25] iteration 2122/ 2384 | consumed samples: 67904 | elapsed time per iteration (ms): 229326.2 | throughput per GPU (TFLOP/s/GPU): 24.8 | MFU 7.94% | FLOPs estimator: workload-aware | logical tokens/s: 73158.4 | valid tokens/s: 73158.4 | physical tokens/s: 74076.3 | loss tokens/s: 57014.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.936467E-01 | lm-global-supervised-tokens: 1.307491E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.855624E-02 | lm-token-buffer-mib: 2.358594E+02 | global_load_balancing_loss: 9.976819E-01 | loss scale: 1.0 | grad norm: 0.663 | params norm: 11148.558 | params rms: 0.060366 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:41:23.474771 | finish at 2026-10-17 19:54:48 + [2026-10-17 03:17:07] iteration 2123/ 2384 | consumed samples: 67936 | elapsed time per iteration (ms): 221842.5 | throughput per GPU (TFLOP/s/GPU): 26.3 | MFU 8.43% | FLOPs estimator: workload-aware | logical tokens/s: 75626.5 | valid tokens/s: 75626.5 | physical tokens/s: 76041.6 | loss tokens/s: 55048.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.048931E-01 | lm-global-supervised-tokens: 1.221198E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.611360E-02 | lm-token-buffer-mib: 2.328420E+02 | global_load_balancing_loss: 9.995933E-01 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11148.526 | params rms: 0.060366 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:05:00.881354 | finish at 2026-10-17 19:22:08 + [2026-10-17 03:20:49] iteration 2124/ 2384 | consumed samples: 67968 | elapsed time per iteration (ms): 221977.4 | throughput per GPU (TFLOP/s/GPU): 24.0 | MFU 7.68% | FLOPs estimator: workload-aware | logical tokens/s: 75580.7 | valid tokens/s: 75580.7 | physical tokens/s: 76106.1 | loss tokens/s: 57935.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.327871E-01 | lm-global-supervised-tokens: 1.286029E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.086130E-02 | lm-token-buffer-mib: 2.325634E+02 | global_load_balancing_loss: 9.992623E-01 | loss scale: 1.0 | grad norm: 0.163 | params norm: 11148.494 | params rms: 0.060365 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:01:54.119058 | finish at 2026-10-17 19:22:43 + [2026-10-17 03:24:29] iteration 2125/ 2384 | consumed samples: 68000 | elapsed time per iteration (ms): 220232.7 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.52% | FLOPs estimator: workload-aware | logical tokens/s: 76179.5 | valid tokens/s: 76179.5 | physical tokens/s: 76643.9 | loss tokens/s: 55532.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.483018E-01 | lm-global-supervised-tokens: 1.223005E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.266322E-02 | lm-token-buffer-mib: 2.334194E+02 | global_load_balancing_loss: 1.001405E+00 | loss scale: 1.0 | grad norm: 0.248 | params norm: 11148.467 | params rms: 0.060365 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:50:40.269143 | finish at 2026-10-17 19:15:09 + [2026-10-17 03:28:11] iteration 2126/ 2384 | consumed samples: 68032 | elapsed time per iteration (ms): 222501.9 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.87% | FLOPs estimator: workload-aware | logical tokens/s: 75402.5 | valid tokens/s: 75402.5 | physical tokens/s: 75911.7 | loss tokens/s: 56167.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.225607E-01 | lm-global-supervised-tokens: 1.249740E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.225131E-02 | lm-token-buffer-mib: 2.330367E+02 | global_load_balancing_loss: 1.000872E+00 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11148.439 | params rms: 0.060365 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:56:45.490189 | finish at 2026-10-17 19:24:57 + [2026-10-17 03:31:53] iteration 2127/ 2384 | consumed samples: 68064 | elapsed time per iteration (ms): 221585.6 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.30% | FLOPs estimator: workload-aware | logical tokens/s: 75714.3 | valid tokens/s: 75714.3 | physical tokens/s: 76471.7 | loss tokens/s: 59488.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.068140E-01 | lm-global-supervised-tokens: 1.318177E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.874967E-02 | lm-token-buffer-mib: 2.342485E+02 | global_load_balancing_loss: 9.999442E-01 | loss scale: 1.0 | grad norm: 0.133 | params norm: 11148.412 | params rms: 0.060365 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:49:07.503402 | finish at 2026-10-17 19:21:00 + [2026-10-17 03:35:33] iteration 2128/ 2384 | consumed samples: 68096 | elapsed time per iteration (ms): 219841.3 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.50% | FLOPs estimator: workload-aware | logical tokens/s: 76315.1 | valid tokens/s: 76315.1 | physical tokens/s: 77376.8 | loss tokens/s: 57789.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.521105E-01 | lm-global-supervised-tokens: 1.270444E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.279986E-02 | lm-token-buffer-mib: 2.332306E+02 | global_load_balancing_loss: 1.001276E+00 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11148.388 | params rms: 0.060365 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:37:59.376038 | finish at 2026-10-17 19:13:32 + [2026-10-17 03:39:14] iteration 2129/ 2384 | consumed samples: 68128 | elapsed time per iteration (ms): 221359.8 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.62% | FLOPs estimator: workload-aware | logical tokens/s: 75791.4 | valid tokens/s: 75791.4 | physical tokens/s: 76862.8 | loss tokens/s: 57598.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.758108E-01 | lm-global-supervised-tokens: 1.275002E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.392590E-02 | lm-token-buffer-mib: 2.337538E+02 | global_load_balancing_loss: 1.000778E+00 | loss scale: 1.0 | grad norm: 0.204 | params norm: 11148.365 | params rms: 0.060365 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:40:46.760091 | finish at 2026-10-17 19:20:01 + [2026-10-17 03:43:17] iteration 2130/ 2384 | consumed samples: 68160 | elapsed time per iteration (ms): 243170.7 | throughput per GPU (TFLOP/s/GPU): 17.7 | MFU 5.68% | FLOPs estimator: workload-aware | logical tokens/s: 68993.5 | valid tokens/s: 68993.5 | physical tokens/s: 69363.1 | loss tokens/s: 52088.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.807925E-01 | lm-global-supervised-tokens: 1.266628E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.761490E-02 | lm-token-buffer-mib: 2.339006E+02 | global_load_balancing_loss: 1.002087E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11148.345 | params rms: 0.060365 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:09:25.347766 | finish at 2026-10-17 20:52:43 + [2026-10-17 03:47:00] iteration 2131/ 2384 | consumed samples: 68192 | elapsed time per iteration (ms): 222774.0 | throughput per GPU (TFLOP/s/GPU): 19.7 | MFU 6.32% | FLOPs estimator: workload-aware | logical tokens/s: 75310.3 | valid tokens/s: 75310.3 | physical tokens/s: 75767.2 | loss tokens/s: 56693.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.459333E-01 | lm-global-supervised-tokens: 1.262973E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.249007E-02 | lm-token-buffer-mib: 2.337645E+02 | global_load_balancing_loss: 9.992173E-01 | loss scale: 1.0 | grad norm: 0.106 | params norm: 11148.325 | params rms: 0.060365 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:39:21.831090 | finish at 2026-10-17 19:26:22 + [2026-10-17 03:50:44] iteration 2132/ 2384 | consumed samples: 68224 | elapsed time per iteration (ms): 223451.3 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.74% | FLOPs estimator: workload-aware | logical tokens/s: 75082.0 | valid tokens/s: 75082.0 | physical tokens/s: 76236.2 | loss tokens/s: 60873.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.915222E-01 | lm-global-supervised-tokens: 1.360224E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.415193E-02 | lm-token-buffer-mib: 2.356952E+02 | global_load_balancing_loss: 9.985188E-01 | loss scale: 1.0 | grad norm: 0.198 | params norm: 11148.303 | params rms: 0.060364 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:38:29.720306 | finish at 2026-10-17 19:29:13 + [2026-10-17 03:54:31] iteration 2133/ 2384 | consumed samples: 68256 | elapsed time per iteration (ms): 227201.7 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.88% | FLOPs estimator: workload-aware | logical tokens/s: 73842.8 | valid tokens/s: 73842.8 | physical tokens/s: 74636.1 | loss tokens/s: 57424.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.528410E-01 | lm-global-supervised-tokens: 1.304692E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.003322E-02 | lm-token-buffer-mib: 2.343966E+02 | global_load_balancing_loss: 9.999650E-01 | loss scale: 1.0 | grad norm: 0.163 | params norm: 11148.280 | params rms: 0.060364 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:50:27.626035 | finish at 2026-10-17 19:44:58 + [2026-10-17 03:58:11] iteration 2134/ 2384 | consumed samples: 68288 | elapsed time per iteration (ms): 220240.6 | throughput per GPU (TFLOP/s/GPU): 23.4 | MFU 7.49% | FLOPs estimator: workload-aware | logical tokens/s: 76176.6 | valid tokens/s: 76176.6 | physical tokens/s: 76803.3 | loss tokens/s: 58637.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.264994E-01 | lm-global-supervised-tokens: 1.291434E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.122977E-02 | lm-token-buffer-mib: 2.337065E+02 | global_load_balancing_loss: 9.990881E-01 | loss scale: 1.0 | grad norm: 0.130 | params norm: 11148.258 | params rms: 0.060364 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:17:40.156465 | finish at 2026-10-17 19:15:51 + [2026-10-17 04:01:57] iteration 2135/ 2384 | consumed samples: 68320 | elapsed time per iteration (ms): 226169.1 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.27% | FLOPs estimator: workload-aware | logical tokens/s: 74179.7 | valid tokens/s: 74179.7 | physical tokens/s: 75739.4 | loss tokens/s: 53840.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.603417E-01 | lm-global-supervised-tokens: 1.217706E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.145657E-02 | lm-token-buffer-mib: 2.361938E+02 | global_load_balancing_loss: 1.000978E+00 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11148.234 | params rms: 0.060364 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:38:36.116538 | finish at 2026-10-17 19:40:33 + [2026-10-17 04:05:43] iteration 2136/ 2384 | consumed samples: 68352 | elapsed time per iteration (ms): 225612.3 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.97% | FLOPs estimator: workload-aware | logical tokens/s: 74362.9 | valid tokens/s: 74362.9 | physical tokens/s: 75031.6 | loss tokens/s: 54038.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.673044E-01 | lm-global-supervised-tokens: 1.219164E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.332909E-02 | lm-token-buffer-mib: 2.344656E+02 | global_load_balancing_loss: 1.001583E+00 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11148.210 | params rms: 0.060364 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:32:31.854401 | finish at 2026-10-17 19:38:15 + [2026-10-17 04:09:23] iteration 2137/ 2384 | consumed samples: 68384 | elapsed time per iteration (ms): 220573.3 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.35% | FLOPs estimator: workload-aware | logical tokens/s: 76061.7 | valid tokens/s: 76061.7 | physical tokens/s: 76672.0 | loss tokens/s: 57777.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.285895E-01 | lm-global-supervised-tokens: 1.274422E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.428471E-02 | lm-token-buffer-mib: 2.331077E+02 | global_load_balancing_loss: 1.000998E+00 | loss scale: 1.0 | grad norm: 0.116 | params norm: 11148.183 | params rms: 0.060364 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:08:01.615613 | finish at 2026-10-17 19:17:25 + [2026-10-17 04:13:08] iteration 2138/ 2384 | consumed samples: 68416 | elapsed time per iteration (ms): 224365.4 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.41% | FLOPs estimator: workload-aware | logical tokens/s: 74776.2 | valid tokens/s: 74776.2 | physical tokens/s: 75882.8 | loss tokens/s: 59149.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.778794E-01 | lm-global-supervised-tokens: 1.327100E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.237923E-02 | lm-token-buffer-mib: 2.363474E+02 | global_load_balancing_loss: 1.000607E+00 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11148.157 | params rms: 0.060364 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:19:53.888125 | finish at 2026-10-17 19:33:02 + [2026-10-17 04:16:48] iteration 2139/ 2384 | consumed samples: 68448 | elapsed time per iteration (ms): 220546.4 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.03% | FLOPs estimator: workload-aware | logical tokens/s: 76071.0 | valid tokens/s: 76071.0 | physical tokens/s: 76662.5 | loss tokens/s: 54740.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.377442E-01 | lm-global-supervised-tokens: 1.207281E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.273732E-02 | lm-token-buffer-mib: 2.332941E+02 | global_load_balancing_loss: 9.997954E-01 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11148.129 | params rms: 0.060363 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:00:33.870938 | finish at 2026-10-17 19:17:22 + [2026-10-17 04:20:54] iteration 2140/ 2384 | consumed samples: 68480 | elapsed time per iteration (ms): 245331.4 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.09% | FLOPs estimator: workload-aware | logical tokens/s: 68385.9 | valid tokens/s: 68385.9 | physical tokens/s: 68856.5 | loss tokens/s: 50956.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.537523E-01 | lm-global-supervised-tokens: 1.250112E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.579161E-02 | lm-token-buffer-mib: 2.339543E+02 | global_load_balancing_loss: 1.000050E+00 | loss scale: 1.0 | grad norm: 0.161 | params norm: 11148.099 | params rms: 0.060363 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:37:40.867921 | finish at 2026-10-17 20:58:34 + [2026-10-17 04:24:34] iteration 2141/ 2384 | consumed samples: 68512 | elapsed time per iteration (ms): 220299.2 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.41% | FLOPs estimator: workload-aware | logical tokens/s: 76156.5 | valid tokens/s: 76156.5 | physical tokens/s: 76421.2 | loss tokens/s: 55997.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.394722E-01 | lm-global-supervised-tokens: 1.233612E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.704532E-02 | lm-token-buffer-mib: 2.324727E+02 | global_load_balancing_loss: 1.000340E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11148.066 | params rms: 0.060363 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:52:12.700226 | finish at 2026-10-17 19:16:47 + [2026-10-17 04:28:15] iteration 2142/ 2384 | consumed samples: 68544 | elapsed time per iteration (ms): 221494.5 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.81% | FLOPs estimator: workload-aware | logical tokens/s: 75745.4 | valid tokens/s: 75745.4 | physical tokens/s: 76380.0 | loss tokens/s: 58341.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.291944E-01 | lm-global-supervised-tokens: 1.292227E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.478996E-02 | lm-token-buffer-mib: 2.332781E+02 | global_load_balancing_loss: 1.000622E+00 | loss scale: 1.0 | grad norm: 0.171 | params norm: 11148.034 | params rms: 0.060363 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:53:21.668808 | finish at 2026-10-17 19:21:37 + [2026-10-17 04:32:00] iteration 2143/ 2384 | consumed samples: 68576 | elapsed time per iteration (ms): 224426.3 | throughput per GPU (TFLOP/s/GPU): 23.5 | MFU 7.53% | FLOPs estimator: workload-aware | logical tokens/s: 74755.9 | valid tokens/s: 74755.9 | physical tokens/s: 75569.3 | loss tokens/s: 57410.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.208773E-01 | lm-global-supervised-tokens: 1.288442E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.676859E-02 | lm-token-buffer-mib: 2.353710E+02 | global_load_balancing_loss: 1.001896E+00 | loss scale: 1.0 | grad norm: 0.108 | params norm: 11148.003 | params rms: 0.060363 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:01:26.747218 | finish at 2026-10-17 19:33:27 + [2026-10-17 04:35:46] iteration 2144/ 2384 | consumed samples: 68608 | elapsed time per iteration (ms): 226589.7 | throughput per GPU (TFLOP/s/GPU): 19.6 | MFU 6.28% | FLOPs estimator: workload-aware | logical tokens/s: 74042.2 | valid tokens/s: 74042.2 | physical tokens/s: 75372.6 | loss tokens/s: 59050.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.498068E-01 | lm-global-supervised-tokens: 1.338028E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.521856E-02 | lm-token-buffer-mib: 2.354907E+02 | global_load_balancing_loss: 9.999157E-01 | loss scale: 1.0 | grad norm: 0.454 | params norm: 11147.972 | params rms: 0.060363 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:06:21.527824 | finish at 2026-10-17 19:42:08 + [2026-10-17 04:39:31] iteration 2145/ 2384 | consumed samples: 68640 | elapsed time per iteration (ms): 224836.0 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.11% | FLOPs estimator: workload-aware | logical tokens/s: 74619.7 | valid tokens/s: 74619.7 | physical tokens/s: 75032.3 | loss tokens/s: 52478.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.423627E-01 | lm-global-supervised-tokens: 1.179905E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.242889E-02 | lm-token-buffer-mib: 2.334920E+02 | global_load_balancing_loss: 1.000239E+00 | loss scale: 1.0 | grad norm: 0.137 | params norm: 11147.940 | params rms: 0.060362 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:55:35.797895 | finish at 2026-10-17 19:35:07 + [2026-10-17 04:43:15] iteration 2146/ 2384 | consumed samples: 68672 | elapsed time per iteration (ms): 223609.3 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.09% | FLOPs estimator: workload-aware | logical tokens/s: 75029.0 | valid tokens/s: 75029.0 | physical tokens/s: 75770.5 | loss tokens/s: 58526.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.723985E-01 | lm-global-supervised-tokens: 1.308702E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.788048E-02 | lm-token-buffer-mib: 2.351387E+02 | global_load_balancing_loss: 1.000571E+00 | loss scale: 1.0 | grad norm: 0.402 | params norm: 11147.905 | params rms: 0.060362 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:46:59.007304 | finish at 2026-10-17 19:30:14 + [2026-10-17 04:46:58] iteration 2147/ 2384 | consumed samples: 68704 | elapsed time per iteration (ms): 223545.6 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.42% | FLOPs estimator: workload-aware | logical tokens/s: 75050.5 | valid tokens/s: 75050.5 | physical tokens/s: 75766.0 | loss tokens/s: 57085.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.218658E-01 | lm-global-supervised-tokens: 1.276115E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.187015E-02 | lm-token-buffer-mib: 2.352521E+02 | global_load_balancing_loss: 9.985206E-01 | loss scale: 1.0 | grad norm: 0.159 | params norm: 11147.872 | params rms: 0.060362 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:43:00.301478 | finish at 2026-10-17 19:29:59 + [2026-10-17 04:50:40] iteration 2148/ 2384 | consumed samples: 68736 | elapsed time per iteration (ms): 221817.4 | throughput per GPU (TFLOP/s/GPU): 20.2 | MFU 6.47% | FLOPs estimator: workload-aware | logical tokens/s: 75635.1 | valid tokens/s: 75635.1 | physical tokens/s: 76493.9 | loss tokens/s: 60624.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.413258E-01 | lm-global-supervised-tokens: 1.344750E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.821183E-02 | lm-token-buffer-mib: 2.340170E+02 | global_load_balancing_loss: 9.997127E-01 | loss scale: 1.0 | grad norm: 0.266 | params norm: 11147.841 | params rms: 0.060362 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:32:28.911684 | finish at 2026-10-17 19:23:09 + [2026-10-17 04:54:24] iteration 2149/ 2384 | consumed samples: 68768 | elapsed time per iteration (ms): 223290.4 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.41% | FLOPs estimator: workload-aware | logical tokens/s: 75136.2 | valid tokens/s: 75136.2 | physical tokens/s: 75897.0 | loss tokens/s: 55925.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.451675E-01 | lm-global-supervised-tokens: 1.248764E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.009572E-02 | lm-token-buffer-mib: 2.354740E+02 | global_load_balancing_loss: 1.000792E+00 | loss scale: 1.0 | grad norm: 0.257 | params norm: 11147.813 | params rms: 0.060362 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:34:33.251514 | finish at 2026-10-17 19:28:57 + [2026-10-17 04:58:29] iteration 2150/ 2384 | consumed samples: 68800 | elapsed time per iteration (ms): 245860.4 | throughput per GPU (TFLOP/s/GPU): 19.2 | MFU 6.14% | FLOPs estimator: workload-aware | logical tokens/s: 68238.7 | valid tokens/s: 68238.7 | physical tokens/s: 69236.3 | loss tokens/s: 50146.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.266447E-01 | lm-global-supervised-tokens: 1.232906E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.883623E-02 | lm-token-buffer-mib: 2.354505E+02 | global_load_balancing_loss: 1.000806E+00 | loss scale: 1.0 | grad norm: 0.203 | params norm: 11147.784 | params rms: 0.060362 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:58:51.326506 | finish at 2026-10-17 20:57:21 + [2026-10-17 05:02:11] iteration 2151/ 2384 | consumed samples: 68832 | elapsed time per iteration (ms): 221328.8 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 7.00% | FLOPs estimator: workload-aware | logical tokens/s: 75802.0 | valid tokens/s: 75802.0 | physical tokens/s: 76670.9 | loss tokens/s: 57856.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.162387E-01 | lm-global-supervised-tokens: 1.280526E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.684859E-02 | lm-token-buffer-mib: 2.328028E+02 | global_load_balancing_loss: 1.000153E+00 | loss scale: 1.0 | grad norm: 0.113 | params norm: 11147.749 | params rms: 0.060361 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:19:29.609503 | finish at 2026-10-17 19:21:40 + [2026-10-17 05:05:54] iteration 2152/ 2384 | consumed samples: 68864 | elapsed time per iteration (ms): 222990.4 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.30% | FLOPs estimator: workload-aware | logical tokens/s: 75237.3 | valid tokens/s: 75237.3 | physical tokens/s: 75608.5 | loss tokens/s: 58883.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.291235E-01 | lm-global-supervised-tokens: 1.313035E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.416144E-02 | lm-token-buffer-mib: 2.325491E+02 | global_load_balancing_loss: 1.000439E+00 | loss scale: 1.0 | grad norm: 0.111 | params norm: 11147.713 | params rms: 0.060361 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:22:13.767508 | finish at 2026-10-17 19:28:07 + [2026-10-17 05:09:40] iteration 2153/ 2384 | consumed samples: 68896 | elapsed time per iteration (ms): 225975.6 | throughput per GPU (TFLOP/s/GPU): 25.9 | MFU 8.30% | FLOPs estimator: workload-aware | logical tokens/s: 74243.4 | valid tokens/s: 74243.4 | physical tokens/s: 74986.1 | loss tokens/s: 55749.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.574647E-01 | lm-global-supervised-tokens: 1.259791E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.228710E-02 | lm-token-buffer-mib: 2.349003E+02 | global_load_balancing_loss: 9.990520E-01 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11147.678 | params rms: 0.060361 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:30:00.371091 | finish at 2026-10-17 19:39:40 + [2026-10-17 05:13:21] iteration 2154/ 2384 | consumed samples: 68928 | elapsed time per iteration (ms): 221369.6 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.58% | FLOPs estimator: workload-aware | logical tokens/s: 75788.2 | valid tokens/s: 75788.2 | physical tokens/s: 76612.2 | loss tokens/s: 56790.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.051144E-01 | lm-global-supervised-tokens: 1.257168E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.751619E-02 | lm-token-buffer-mib: 2.354345E+02 | global_load_balancing_loss: 9.996211E-01 | loss scale: 1.0 | grad norm: 0.154 | params norm: 11147.642 | params rms: 0.060361 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:08:35.002365 | finish at 2026-10-17 19:21:56 + [2026-10-17 05:17:03] iteration 2155/ 2384 | consumed samples: 68960 | elapsed time per iteration (ms): 221478.1 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.75% | FLOPs estimator: workload-aware | logical tokens/s: 75751.1 | valid tokens/s: 75751.1 | physical tokens/s: 76510.0 | loss tokens/s: 55162.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.846218E-01 | lm-global-supervised-tokens: 1.221738E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.650252E-02 | lm-token-buffer-mib: 2.326612E+02 | global_load_balancing_loss: 1.000385E+00 | loss scale: 1.0 | grad norm: 0.112 | params norm: 11147.605 | params rms: 0.060361 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:05:18.480219 | finish at 2026-10-17 19:22:21 + [2026-10-17 05:20:42] iteration 2156/ 2384 | consumed samples: 68992 | elapsed time per iteration (ms): 219860.2 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.56% | FLOPs estimator: workload-aware | logical tokens/s: 76308.5 | valid tokens/s: 76308.5 | physical tokens/s: 76823.5 | loss tokens/s: 57325.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.526750E-01 | lm-global-supervised-tokens: 1.260353E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.357187E-02 | lm-token-buffer-mib: 2.330131E+02 | global_load_balancing_loss: 1.002036E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11147.569 | params rms: 0.060360 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:55:28.117973 | finish at 2026-10-17 19:16:10 + [2026-10-17 05:24:28] iteration 2157/ 2384 | consumed samples: 69024 | elapsed time per iteration (ms): 225330.7 | throughput per GPU (TFLOP/s/GPU): 24.6 | MFU 7.88% | FLOPs estimator: workload-aware | logical tokens/s: 74455.9 | valid tokens/s: 74455.9 | physical tokens/s: 75199.8 | loss tokens/s: 52513.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.085759E-01 | lm-global-supervised-tokens: 1.183287E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.820103E-02 | lm-token-buffer-mib: 2.360838E+02 | global_load_balancing_loss: 1.002743E+00 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11147.536 | params rms: 0.060360 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:12:30.065959 | finish at 2026-10-17 19:36:58 + [2026-10-17 05:28:09] iteration 2158/ 2384 | consumed samples: 69056 | elapsed time per iteration (ms): 221311.3 | throughput per GPU (TFLOP/s/GPU): 23.9 | MFU 7.66% | FLOPs estimator: workload-aware | logical tokens/s: 75808.0 | valid tokens/s: 75808.0 | physical tokens/s: 76599.2 | loss tokens/s: 58918.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.774217E-01 | lm-global-supervised-tokens: 1.303935E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.437097E-02 | lm-token-buffer-mib: 2.331714E+02 | global_load_balancing_loss: 1.001462E+00 | loss scale: 1.0 | grad norm: 0.323 | params norm: 11147.505 | params rms: 0.060360 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:53:36.345996 | finish at 2026-10-17 19:21:45 + [2026-10-17 05:31:53] iteration 2159/ 2384 | consumed samples: 69088 | elapsed time per iteration (ms): 224039.9 | throughput per GPU (TFLOP/s/GPU): 25.8 | MFU 8.28% | FLOPs estimator: workload-aware | logical tokens/s: 74884.8 | valid tokens/s: 74884.8 | physical tokens/s: 75371.1 | loss tokens/s: 53043.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.557179E-01 | lm-global-supervised-tokens: 1.188389E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.642691E-02 | lm-token-buffer-mib: 2.334741E+02 | global_load_balancing_loss: 1.000400E+00 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11147.474 | params rms: 0.060360 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:00:08.975905 | finish at 2026-10-17 19:32:02 + [2026-10-17 05:36:02] iteration 2160/ 2384 | consumed samples: 69120 | elapsed time per iteration (ms): 248554.1 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.57% | FLOPs estimator: workload-aware | logical tokens/s: 67499.1 | valid tokens/s: 67499.1 | physical tokens/s: 67736.7 | loss tokens/s: 48372.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.129540E-01 | lm-global-supervised-tokens: 1.202306E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.629336E-02 | lm-token-buffer-mib: 2.330240E+02 | global_load_balancing_loss: 9.980429E-01 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11147.442 | params rms: 0.060360 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:27:56.122681 | finish at 2026-10-17 21:03:58 +times across ranks (ms): + save-checkpoint: + rank 0: 44933.11 + rank 1: 44933.12 + rank 2: 44933.07 + rank 3: 44933.07 + rank 4: 44933.09 + rank 5: 44932.99 + rank 6: 44933.11 + rank 7: 44933.12 + rank 8: 44591.45 + rank 9: 44591.39 + rank 10: 44591.44 + rank 11: 44591.41 + rank 12: 44591.41 + rank 13: 44591.41 + rank 14: 44591.37 + rank 15: 44591.44 + rank 16: 44591.36 + rank 17: 44591.38 + rank 18: 44591.36 + rank 19: 44591.41 + rank 20: 44591.39 + rank 21: 44591.39 + rank 22: 44591.39 + rank 23: 44591.35 + rank 24: 44590.68 + rank 25: 44590.68 + rank 26: 44590.68 + rank 27: 44590.54 + rank 28: 44590.60 + rank 29: 44590.67 + rank 30: 44590.65 + rank 31: 44590.61 + rank 32: 44590.57 + rank 33: 44590.49 + rank 34: 44590.47 + rank 35: 44590.48 + rank 36: 44590.44 + rank 37: 44590.48 + rank 38: 44590.42 + rank 39: 44590.43 + rank 40: 44591.60 + rank 41: 44591.63 + rank 42: 44591.60 + rank 43: 44591.59 + rank 44: 44591.59 + rank 45: 44591.61 + rank 46: 44591.60 + rank 47: 44591.61 + rank 48: 44590.96 + rank 49: 44591.12 + rank 50: 44591.09 + rank 51: 44591.08 + rank 52: 44590.96 + rank 53: 44591.01 + rank 54: 44591.05 + rank 55: 44590.98 + rank 56: 44590.46 + rank 57: 44590.52 + rank 58: 44590.54 + rank 59: 44590.51 + rank 60: 44590.52 + rank 61: 44590.47 + rank 62: 44590.46 + rank 63: 44590.46 + [2026-10-17 05:40:34] iteration 2161/ 2384 | consumed samples: 69152 | elapsed time per iteration (ms): 222398.9 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.60% | FLOPs estimator: workload-aware | logical tokens/s: 75437.3 | valid tokens/s: 75437.3 | physical tokens/s: 75974.2 | loss tokens/s: 59919.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.573752E-01 | lm-global-supervised-tokens: 1.332598E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.456062E-01 | lm-token-buffer-mib: 2.332933E+02 | global_load_balancing_loss: 1.000070E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11147.407 | params rms: 0.060360 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:46:34.950241 | finish at 2026-10-17 19:27:09 + [2026-10-17 05:44:19] iteration 2162/ 2384 | consumed samples: 69184 | elapsed time per iteration (ms): 224660.7 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.07% | FLOPs estimator: workload-aware | logical tokens/s: 74677.9 | valid tokens/s: 74677.9 | physical tokens/s: 75378.5 | loss tokens/s: 54941.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.353574E-01 | lm-global-supervised-tokens: 1.234317E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.380177E-02 | lm-token-buffer-mib: 2.343587E+02 | global_load_balancing_loss: 9.993462E-01 | loss scale: 1.0 | grad norm: 0.167 | params norm: 11147.373 | params rms: 0.060359 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:51:14.682564 | finish at 2026-10-17 19:35:33 + [2026-10-17 05:48:00] iteration 2163/ 2384 | consumed samples: 69216 | elapsed time per iteration (ms): 221351.5 | throughput per GPU (TFLOP/s/GPU): 25.7 | MFU 8.25% | FLOPs estimator: workload-aware | logical tokens/s: 75794.2 | valid tokens/s: 75794.2 | physical tokens/s: 76603.4 | loss tokens/s: 55679.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.915810E-01 | lm-global-supervised-tokens: 1.232471E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.567426E-02 | lm-token-buffer-mib: 2.348883E+02 | global_load_balancing_loss: 9.997872E-01 | loss scale: 1.0 | grad norm: 0.217 | params norm: 11147.339 | params rms: 0.060359 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:35:18.688410 | finish at 2026-10-17 19:23:19 + [2026-10-17 05:51:40] iteration 2164/ 2384 | consumed samples: 69248 | elapsed time per iteration (ms): 220382.2 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.11% | FLOPs estimator: workload-aware | logical tokens/s: 76127.7 | valid tokens/s: 76127.7 | physical tokens/s: 77212.2 | loss tokens/s: 56110.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.219574E-01 | lm-global-supervised-tokens: 1.236578E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.441973E-02 | lm-token-buffer-mib: 2.329455E+02 | global_load_balancing_loss: 9.991992E-01 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11147.303 | params rms: 0.060359 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:28:04.087253 | finish at 2026-10-17 19:19:45 + [2026-10-17 05:55:24] iteration 2165/ 2384 | consumed samples: 69280 | elapsed time per iteration (ms): 223222.2 | throughput per GPU (TFLOP/s/GPU): 23.4 | MFU 7.51% | FLOPs estimator: workload-aware | logical tokens/s: 75159.1 | valid tokens/s: 75159.1 | physical tokens/s: 75661.9 | loss tokens/s: 56242.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.211272E-01 | lm-global-supervised-tokens: 1.255463E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.547902E-02 | lm-token-buffer-mib: 2.335716E+02 | global_load_balancing_loss: 1.001608E+00 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11147.266 | params rms: 0.060359 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:34:45.653584 | finish at 2026-10-17 19:30:09 + [2026-10-17 05:59:07] iteration 2166/ 2384 | consumed samples: 69312 | elapsed time per iteration (ms): 223471.2 | throughput per GPU (TFLOP/s/GPU): 23.9 | MFU 7.65% | FLOPs estimator: workload-aware | logical tokens/s: 75075.5 | valid tokens/s: 75075.5 | physical tokens/s: 75862.2 | loss tokens/s: 57569.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.902965E-01 | lm-global-supervised-tokens: 1.286506E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.319472E-02 | lm-token-buffer-mib: 2.356621E+02 | global_load_balancing_loss: 1.002343E+00 | loss scale: 1.0 | grad norm: 0.276 | params norm: 11147.232 | params rms: 0.060359 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:31:56.712190 | finish at 2026-10-17 19:31:04 + [2026-10-17 06:02:48] iteration 2167/ 2384 | consumed samples: 69344 | elapsed time per iteration (ms): 220964.9 | throughput per GPU (TFLOP/s/GPU): 19.9 | MFU 6.39% | FLOPs estimator: workload-aware | logical tokens/s: 75927.0 | valid tokens/s: 75927.0 | physical tokens/s: 76435.1 | loss tokens/s: 51332.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.228633E-01 | lm-global-supervised-tokens: 1.134261E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.297346E-02 | lm-token-buffer-mib: 2.330274E+02 | global_load_balancing_loss: 1.001119E+00 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11147.200 | params rms: 0.060358 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:19:09.393703 | finish at 2026-10-17 19:21:57 + [2026-10-17 06:06:33] iteration 2168/ 2384 | consumed samples: 69376 | elapsed time per iteration (ms): 225230.0 | throughput per GPU (TFLOP/s/GPU): 23.4 | MFU 7.50% | FLOPs estimator: workload-aware | logical tokens/s: 74489.2 | valid tokens/s: 74489.2 | physical tokens/s: 75159.9 | loss tokens/s: 57665.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.874243E-01 | lm-global-supervised-tokens: 1.298796E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.792341E-02 | lm-token-buffer-mib: 2.342571E+02 | global_load_balancing_loss: 9.998615E-01 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11147.165 | params rms: 0.060358 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:30:49.685257 | finish at 2026-10-17 19:37:23 + [2026-10-17 06:10:15] iteration 2169/ 2384 | consumed samples: 69408 | elapsed time per iteration (ms): 221782.7 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.80% | FLOPs estimator: workload-aware | logical tokens/s: 75647.0 | valid tokens/s: 75647.0 | physical tokens/s: 76310.2 | loss tokens/s: 51744.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.094441E-01 | lm-global-supervised-tokens: 1.147600E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.004839E-02 | lm-token-buffer-mib: 2.326332E+02 | global_load_balancing_loss: 1.000882E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11147.131 | params rms: 0.060358 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:14:43.290765 | finish at 2026-10-17 19:24:58 + [2026-10-17 06:14:19] iteration 2170/ 2384 | consumed samples: 69440 | elapsed time per iteration (ms): 244144.6 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.17% | FLOPs estimator: workload-aware | logical tokens/s: 68718.3 | valid tokens/s: 68718.3 | physical tokens/s: 69043.7 | loss tokens/s: 51874.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.617446E-01 | lm-global-supervised-tokens: 1.266484E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.171426E-02 | lm-token-buffer-mib: 2.334626E+02 | global_load_balancing_loss: 9.997509E-01 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11147.099 | params rms: 0.060358 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:30:46.939688 | finish at 2026-10-17 20:45:06 + [2026-10-17 06:18:02] iteration 2171/ 2384 | consumed samples: 69472 | elapsed time per iteration (ms): 223237.0 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.76% | FLOPs estimator: workload-aware | logical tokens/s: 75154.3 | valid tokens/s: 75154.2 | physical tokens/s: 75844.9 | loss tokens/s: 58839.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.617413E-01 | lm-global-supervised-tokens: 1.313510E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.471180E-02 | lm-token-buffer-mib: 2.345193E+02 | global_load_balancing_loss: 9.996725E-01 | loss scale: 1.0 | grad norm: 0.157 | params norm: 11147.065 | params rms: 0.060358 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:12:29.488717 | finish at 2026-10-17 19:30:32 + [2026-10-17 06:21:48] iteration 2172/ 2384 | consumed samples: 69504 | elapsed time per iteration (ms): 225586.9 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.70% | FLOPs estimator: workload-aware | logical tokens/s: 74371.4 | valid tokens/s: 74371.4 | physical tokens/s: 74951.6 | loss tokens/s: 56855.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.479285E-01 | lm-global-supervised-tokens: 1.282592E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.524851E-02 | lm-token-buffer-mib: 2.341304E+02 | global_load_balancing_loss: 9.992290E-01 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11147.034 | params rms: 0.060358 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:17:04.416178 | finish at 2026-10-17 19:38:52 + [2026-10-17 06:25:31] iteration 2173/ 2384 | consumed samples: 69536 | elapsed time per iteration (ms): 222811.1 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.14% | FLOPs estimator: workload-aware | logical tokens/s: 75297.7 | valid tokens/s: 75297.7 | physical tokens/s: 76103.1 | loss tokens/s: 59273.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.347541E-01 | lm-global-supervised-tokens: 1.320686E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.776005E-02 | lm-token-buffer-mib: 2.348061E+02 | global_load_balancing_loss: 1.000738E+00 | loss scale: 1.0 | grad norm: 0.118 | params norm: 11147.003 | params rms: 0.060357 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:03:33.139083 | finish at 2026-10-17 19:29:04 + [2026-10-17 06:29:15] iteration 2174/ 2384 | consumed samples: 69568 | elapsed time per iteration (ms): 223817.0 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.45% | FLOPs estimator: workload-aware | logical tokens/s: 74959.5 | valid tokens/s: 74959.5 | physical tokens/s: 75612.9 | loss tokens/s: 60259.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.150890E-01 | lm-global-supervised-tokens: 1.348700E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.223668E-02 | lm-token-buffer-mib: 2.345727E+02 | global_load_balancing_loss: 1.000470E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11146.973 | params rms: 0.060357 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:03:21.567478 | finish at 2026-10-17 19:32:36 + [2026-10-17 06:33:01] iteration 2175/ 2384 | consumed samples: 69600 | elapsed time per iteration (ms): 226088.7 | throughput per GPU (TFLOP/s/GPU): 24.8 | MFU 7.94% | FLOPs estimator: workload-aware | logical tokens/s: 74206.1 | valid tokens/s: 74206.1 | physical tokens/s: 74717.3 | loss tokens/s: 58073.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.228748E-01 | lm-global-supervised-tokens: 1.312986E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.809041E-02 | lm-token-buffer-mib: 2.328301E+02 | global_load_balancing_loss: 1.000132E+00 | loss scale: 1.0 | grad norm: 0.105 | params norm: 11146.940 | params rms: 0.060357 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:07:32.547032 | finish at 2026-10-17 19:40:33 + [2026-10-17 06:36:42] iteration 2176/ 2384 | consumed samples: 69632 | elapsed time per iteration (ms): 220766.4 | throughput per GPU (TFLOP/s/GPU): 19.4 | MFU 6.22% | FLOPs estimator: workload-aware | logical tokens/s: 75995.3 | valid tokens/s: 75995.3 | physical tokens/s: 76502.1 | loss tokens/s: 56596.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.809907E-01 | lm-global-supervised-tokens: 1.249450E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.287628E-02 | lm-token-buffer-mib: 2.326867E+02 | global_load_balancing_loss: 9.997790E-01 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11146.906 | params rms: 0.060357 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:45:19.407452 | finish at 2026-10-17 19:22:01 + [2026-10-17 06:40:23] iteration 2177/ 2384 | consumed samples: 69664 | elapsed time per iteration (ms): 221097.4 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.44% | FLOPs estimator: workload-aware | logical tokens/s: 75881.4 | valid tokens/s: 75881.4 | physical tokens/s: 76340.4 | loss tokens/s: 59404.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.077760E-01 | lm-global-supervised-tokens: 1.313426E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.195105E-02 | lm-token-buffer-mib: 2.332655E+02 | global_load_balancing_loss: 1.001020E+00 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11146.873 | params rms: 0.060357 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:42:47.154091 | finish at 2026-10-17 19:23:10 + [2026-10-17 06:44:07] iteration 2178/ 2384 | consumed samples: 69696 | elapsed time per iteration (ms): 224555.4 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.23% | FLOPs estimator: workload-aware | logical tokens/s: 74713.0 | valid tokens/s: 74713.0 | physical tokens/s: 75849.1 | loss tokens/s: 53720.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.199715E-01 | lm-global-supervised-tokens: 1.206315E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.816718E-02 | lm-token-buffer-mib: 2.356491E+02 | global_load_balancing_loss: 1.001381E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11146.842 | params rms: 0.060357 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:50:58.406878 | finish at 2026-10-17 19:35:06 + [2026-10-17 06:47:49] iteration 2179/ 2384 | consumed samples: 69728 | elapsed time per iteration (ms): 221984.0 | throughput per GPU (TFLOP/s/GPU): 23.5 | MFU 7.52% | FLOPs estimator: workload-aware | logical tokens/s: 75578.3 | valid tokens/s: 75578.3 | physical tokens/s: 76333.3 | loss tokens/s: 58860.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.725064E-01 | lm-global-supervised-tokens: 1.306618E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.256739E-02 | lm-token-buffer-mib: 2.350962E+02 | global_load_balancing_loss: 1.001566E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11146.813 | params rms: 0.060356 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:38:26.714177 | finish at 2026-10-17 19:26:16 + [2026-10-17 06:51:51] iteration 2180/ 2384 | consumed samples: 69760 | elapsed time per iteration (ms): 241338.0 | throughput per GPU (TFLOP/s/GPU): 18.5 | MFU 5.94% | FLOPs estimator: workload-aware | logical tokens/s: 69517.4 | valid tokens/s: 69517.4 | physical tokens/s: 70312.3 | loss tokens/s: 53064.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.966886E-01 | lm-global-supervised-tokens: 1.280653E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.587484E-02 | lm-token-buffer-mib: 2.327517E+02 | global_load_balancing_loss: 1.001271E+00 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11146.781 | params rms: 0.060356 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:40:32.956438 | finish at 2026-10-17 20:32:23 + [2026-10-17 06:55:30] iteration 2181/ 2384 | consumed samples: 69792 | elapsed time per iteration (ms): 219642.2 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.50% | FLOPs estimator: workload-aware | logical tokens/s: 76384.2 | valid tokens/s: 76384.2 | physical tokens/s: 76924.0 | loss tokens/s: 55902.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.251388E-01 | lm-global-supervised-tokens: 1.227857E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.987297E-02 | lm-token-buffer-mib: 2.338799E+02 | global_load_balancing_loss: 1.000421E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11146.748 | params rms: 0.060356 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:23:07.358177 | finish at 2026-10-17 19:18:38 + [2026-10-17 06:59:13] iteration 2182/ 2384 | consumed samples: 69824 | elapsed time per iteration (ms): 223237.2 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.76% | FLOPs estimator: workload-aware | logical tokens/s: 75154.2 | valid tokens/s: 75154.2 | physical tokens/s: 76333.6 | loss tokens/s: 53936.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.126949E-01 | lm-global-supervised-tokens: 1.204052E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.296862E-02 | lm-token-buffer-mib: 2.339314E+02 | global_load_balancing_loss: 9.993678E-01 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11146.716 | params rms: 0.060356 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:31:33.920665 | finish at 2026-10-17 19:30:47 + [2026-10-17 07:02:55] iteration 2183/ 2384 | consumed samples: 69856 | elapsed time per iteration (ms): 221328.9 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.25% | FLOPs estimator: workload-aware | logical tokens/s: 75802.1 | valid tokens/s: 75802.1 | physical tokens/s: 76667.9 | loss tokens/s: 52246.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.635703E-01 | lm-global-supervised-tokens: 1.156362E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.654268E-02 | lm-token-buffer-mib: 2.332247E+02 | global_load_balancing_loss: 9.995003E-01 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11146.682 | params rms: 0.060356 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:21:27.114095 | finish at 2026-10-17 19:24:22 + [2026-10-17 07:06:37] iteration 2184/ 2384 | consumed samples: 69888 | elapsed time per iteration (ms): 222390.4 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.55% | FLOPs estimator: workload-aware | logical tokens/s: 75440.3 | valid tokens/s: 75440.3 | physical tokens/s: 75936.2 | loss tokens/s: 56289.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.616614E-01 | lm-global-supervised-tokens: 1.251828E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.604072E-02 | lm-token-buffer-mib: 2.342942E+02 | global_load_balancing_loss: 9.996681E-01 | loss scale: 1.0 | grad norm: 0.139 | params norm: 11146.648 | params rms: 0.060355 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:21:18.082275 | finish at 2026-10-17 19:27:55 + [2026-10-17 07:10:24] iteration 2185/ 2384 | consumed samples: 69920 | elapsed time per iteration (ms): 227182.1 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.56% | FLOPs estimator: workload-aware | logical tokens/s: 73849.0 | valid tokens/s: 73849.0 | physical tokens/s: 74730.1 | loss tokens/s: 53776.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.197073E-01 | lm-global-supervised-tokens: 1.221709E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.233675E-02 | lm-token-buffer-mib: 2.369309E+02 | global_load_balancing_loss: 9.997460E-01 | loss scale: 1.0 | grad norm: 0.103 | params norm: 11146.615 | params rms: 0.060355 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:33:29.230510 | finish at 2026-10-17 19:43:54 + [2026-10-17 07:14:09] iteration 2186/ 2384 | consumed samples: 69952 | elapsed time per iteration (ms): 224221.7 | throughput per GPU (TFLOP/s/GPU): 18.7 | MFU 6.00% | FLOPs estimator: workload-aware | logical tokens/s: 74824.2 | valid tokens/s: 74824.2 | physical tokens/s: 75338.9 | loss tokens/s: 55416.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.246865E-01 | lm-global-supervised-tokens: 1.242550E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.920677E-02 | lm-token-buffer-mib: 2.336578E+02 | global_load_balancing_loss: 9.989558E-01 | loss scale: 1.0 | grad norm: 0.197 | params norm: 11146.585 | params rms: 0.060355 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:19:55.892625 | finish at 2026-10-17 19:34:04 + [2026-10-17 07:17:50] iteration 2187/ 2384 | consumed samples: 69984 | elapsed time per iteration (ms): 221949.4 | throughput per GPU (TFLOP/s/GPU): 25.2 | MFU 8.07% | FLOPs estimator: workload-aware | logical tokens/s: 75590.1 | valid tokens/s: 75590.1 | physical tokens/s: 76035.5 | loss tokens/s: 62883.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.109573E-01 | lm-global-supervised-tokens: 1.395691E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.585573E-02 | lm-token-buffer-mib: 2.328819E+02 | global_load_balancing_loss: 1.000284E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11146.551 | params rms: 0.060355 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:08:44.037802 | finish at 2026-10-17 19:26:35 + [2026-10-17 07:21:31] iteration 2188/ 2384 | consumed samples: 70016 | elapsed time per iteration (ms): 220326.0 | throughput per GPU (TFLOP/s/GPU): 26.5 | MFU 8.51% | FLOPs estimator: workload-aware | logical tokens/s: 76147.2 | valid tokens/s: 76147.2 | physical tokens/s: 76379.3 | loss tokens/s: 57214.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.383259E-01 | lm-global-supervised-tokens: 1.260586E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.255203E-02 | lm-token-buffer-mib: 2.324343E+02 | global_load_balancing_loss: 1.001011E+00 | loss scale: 1.0 | grad norm: 0.137 | params norm: 11146.516 | params rms: 0.060355 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:59:43.896883 | finish at 2026-10-17 19:21:15 + [2026-10-17 07:25:16] iteration 2189/ 2384 | consumed samples: 70048 | elapsed time per iteration (ms): 225530.1 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.45% | FLOPs estimator: workload-aware | logical tokens/s: 74390.1 | valid tokens/s: 74390.1 | physical tokens/s: 75679.1 | loss tokens/s: 56139.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.049320E-01 | lm-global-supervised-tokens: 1.266123E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.640046E-02 | lm-token-buffer-mib: 2.367508E+02 | global_load_balancing_loss: 1.001908E+00 | loss scale: 1.0 | grad norm: 0.115 | params norm: 11146.485 | params rms: 0.060355 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:12:58.363338 | finish at 2026-10-17 19:38:15 + [2026-10-17 07:29:23] iteration 2190/ 2384 | consumed samples: 70080 | elapsed time per iteration (ms): 246418.8 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.66% | FLOPs estimator: workload-aware | logical tokens/s: 68084.0 | valid tokens/s: 68084.0 | physical tokens/s: 69162.8 | loss tokens/s: 53289.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.178916E-01 | lm-global-supervised-tokens: 1.313153E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.822988E-02 | lm-token-buffer-mib: 2.339647E+02 | global_load_balancing_loss: 1.002005E+00 | loss scale: 1.0 | grad norm: 0.165 | params norm: 11146.452 | params rms: 0.060354 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:16:45.251940 | finish at 2026-10-17 20:46:08 + [2026-10-17 07:33:07] iteration 2191/ 2384 | consumed samples: 70112 | elapsed time per iteration (ms): 223844.8 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.74% | FLOPs estimator: workload-aware | logical tokens/s: 74950.1 | valid tokens/s: 74950.0 | physical tokens/s: 75430.6 | loss tokens/s: 52821.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.316096E-01 | lm-global-supervised-tokens: 1.182369E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.369032E-02 | lm-token-buffer-mib: 2.338234E+02 | global_load_balancing_loss: 1.000518E+00 | loss scale: 1.0 | grad norm: 0.354 | params norm: 11146.418 | params rms: 0.060354 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:00:02.037564 | finish at 2026-10-17 19:33:09 + [2026-10-17 07:36:47] iteration 2192/ 2384 | consumed samples: 70144 | elapsed time per iteration (ms): 220404.7 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.07% | FLOPs estimator: workload-aware | logical tokens/s: 76120.0 | valid tokens/s: 76120.0 | physical tokens/s: 76590.7 | loss tokens/s: 55091.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.661346E-01 | lm-global-supervised-tokens: 1.214252E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.661817E-02 | lm-token-buffer-mib: 2.334734E+02 | global_load_balancing_loss: 1.000105E+00 | loss scale: 1.0 | grad norm: 0.207 | params norm: 11146.384 | params rms: 0.060354 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:45:17.707901 | finish at 2026-10-17 19:22:05 + [2026-10-17 07:40:29] iteration 2193/ 2384 | consumed samples: 70176 | elapsed time per iteration (ms): 222132.1 | throughput per GPU (TFLOP/s/GPU): 20.5 | MFU 6.58% | FLOPs estimator: workload-aware | logical tokens/s: 75528.0 | valid tokens/s: 75528.0 | physical tokens/s: 76242.1 | loss tokens/s: 58955.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.740405E-01 | lm-global-supervised-tokens: 1.309586E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.216126E-02 | lm-token-buffer-mib: 2.351270E+02 | global_load_balancing_loss: 1.000752E+00 | loss scale: 1.0 | grad norm: 0.178 | params norm: 11146.351 | params rms: 0.060354 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:47:07.222696 | finish at 2026-10-17 19:27:36 + [2026-10-17 07:44:08] iteration 2194/ 2384 | consumed samples: 70208 | elapsed time per iteration (ms): 219242.5 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.96% | FLOPs estimator: workload-aware | logical tokens/s: 76523.4 | valid tokens/s: 76523.4 | physical tokens/s: 77016.0 | loss tokens/s: 59156.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.240109E-01 | lm-global-supervised-tokens: 1.296967E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.871151E-02 | lm-token-buffer-mib: 2.335391E+02 | global_load_balancing_loss: 1.001091E+00 | loss scale: 1.0 | grad norm: 0.235 | params norm: 11146.318 | params rms: 0.060354 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:34:16.078365 | finish at 2026-10-17 19:18:24 + [2026-10-17 07:47:51] iteration 2195/ 2384 | consumed samples: 70240 | elapsed time per iteration (ms): 222999.7 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.10% | FLOPs estimator: workload-aware | logical tokens/s: 75234.1 | valid tokens/s: 75234.1 | physical tokens/s: 75660.4 | loss tokens/s: 54719.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.608677E-01 | lm-global-supervised-tokens: 1.220234E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.536903E-02 | lm-token-buffer-mib: 2.336269E+02 | global_load_balancing_loss: 9.995317E-01 | loss scale: 1.0 | grad norm: 0.139 | params norm: 11146.289 | params rms: 0.060354 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:42:26.942187 | finish at 2026-10-17 19:30:18 + [2026-10-17 07:51:33] iteration 2196/ 2384 | consumed samples: 70272 | elapsed time per iteration (ms): 221951.6 | throughput per GPU (TFLOP/s/GPU): 24.4 | MFU 7.81% | FLOPs estimator: workload-aware | logical tokens/s: 75589.4 | valid tokens/s: 75589.4 | physical tokens/s: 76422.0 | loss tokens/s: 62848.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.764462E-01 | lm-global-supervised-tokens: 1.394940E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.251327E-02 | lm-token-buffer-mib: 2.350287E+02 | global_load_balancing_loss: 1.000845E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11146.257 | params rms: 0.060353 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:35:26.906507 | finish at 2026-10-17 19:27:00 + [2026-10-17 07:55:15] iteration 2197/ 2384 | consumed samples: 70304 | elapsed time per iteration (ms): 221781.5 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.84% | FLOPs estimator: workload-aware | logical tokens/s: 75647.4 | valid tokens/s: 75647.4 | physical tokens/s: 76794.6 | loss tokens/s: 59224.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.212556E-01 | lm-global-supervised-tokens: 1.313479E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.884400E-02 | lm-token-buffer-mib: 2.345392E+02 | global_load_balancing_loss: 1.000305E+00 | loss scale: 1.0 | grad norm: 0.288 | params norm: 11146.226 | params rms: 0.060353 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:31:13.146984 | finish at 2026-10-17 19:26:28 + [2026-10-17 07:59:00] iteration 2198/ 2384 | consumed samples: 70336 | elapsed time per iteration (ms): 224392.7 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.23% | FLOPs estimator: workload-aware | logical tokens/s: 74767.2 | valid tokens/s: 74767.2 | physical tokens/s: 75297.7 | loss tokens/s: 55198.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.265125E-01 | lm-global-supervised-tokens: 1.238605E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.622158E-02 | lm-token-buffer-mib: 2.334822E+02 | global_load_balancing_loss: 1.001045E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11146.194 | params rms: 0.060353 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:35:37.044010 | finish at 2026-10-17 19:34:37 + [2026-10-17 08:02:44] iteration 2199/ 2384 | consumed samples: 70368 | elapsed time per iteration (ms): 224495.8 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.64% | FLOPs estimator: workload-aware | logical tokens/s: 74732.8 | valid tokens/s: 74732.8 | physical tokens/s: 75618.9 | loss tokens/s: 60589.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.406842E-01 | lm-global-supervised-tokens: 1.360215E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.507245E-02 | lm-token-buffer-mib: 2.350757E+02 | global_load_balancing_loss: 9.995400E-01 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11146.162 | params rms: 0.060353 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:32:11.728384 | finish at 2026-10-17 19:34:56 + [2026-10-17 08:06:47] iteration 2200/ 2384 | consumed samples: 70400 | elapsed time per iteration (ms): 242917.0 | throughput per GPU (TFLOP/s/GPU): 18.4 | MFU 5.90% | FLOPs estimator: workload-aware | logical tokens/s: 69065.5 | valid tokens/s: 69065.5 | physical tokens/s: 69648.1 | loss tokens/s: 53025.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.898910E-01 | lm-global-supervised-tokens: 1.288089E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.350166E-02 | lm-token-buffer-mib: 2.342648E+02 | global_load_balancing_loss: 9.993525E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11146.129 | params rms: 0.060353 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:24:56.733099 | finish at 2026-10-17 20:31:44 +times across ranks (ms): + save-checkpoint: + rank 0: 25276.38 + rank 1: 25276.38 + rank 2: 25276.42 + rank 3: 25276.41 + rank 4: 25276.37 + rank 5: 25276.41 + rank 6: 25276.41 + rank 7: 25276.40 + rank 8: 25276.92 + rank 9: 25276.96 + rank 10: 25276.97 + rank 11: 25276.91 + rank 12: 25276.96 + rank 13: 25276.92 + rank 14: 25276.94 + rank 15: 25276.93 + rank 16: 25276.42 + rank 17: 25276.56 + rank 18: 25276.59 + rank 19: 25276.56 + rank 20: 25276.54 + rank 21: 25276.53 + rank 22: 25276.54 + rank 23: 25276.54 + rank 24: 25276.34 + rank 25: 25276.32 + rank 26: 25276.31 + rank 27: 25276.30 + rank 28: 25276.32 + rank 29: 25276.62 + rank 30: 25276.33 + rank 31: 25276.23 + rank 32: 25279.29 + rank 33: 25279.32 + rank 34: 25279.33 + rank 35: 25279.25 + rank 36: 25279.09 + rank 37: 25279.08 + rank 38: 25279.21 + rank 39: 25279.19 + rank 40: 25277.04 + rank 41: 25277.03 + rank 42: 25277.03 + rank 43: 25277.03 + rank 44: 25277.01 + rank 45: 25277.05 + rank 46: 25277.01 + rank 47: 25277.01 + rank 48: 25276.92 + rank 49: 25276.76 + rank 50: 25276.94 + rank 51: 25276.90 + rank 52: 25276.87 + rank 53: 25276.87 + rank 54: 25276.87 + rank 55: 25276.88 + rank 56: 25276.98 + rank 57: 25277.01 + rank 58: 25277.01 + rank 59: 25277.01 + rank 60: 25276.99 + rank 61: 25276.98 + rank 62: 25276.99 + rank 63: 25276.96 + [2026-10-17 08:10:52] iteration 2201/ 2384 | consumed samples: 70432 | elapsed time per iteration (ms): 219237.1 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.68% | FLOPs estimator: workload-aware | logical tokens/s: 76525.4 | valid tokens/s: 76525.4 | physical tokens/s: 77003.3 | loss tokens/s: 59764.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.449117E-01 | lm-global-supervised-tokens: 1.310250E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 6.084830E-01 | lm-token-buffer-mib: 2.333181E+02 | global_load_balancing_loss: 1.000525E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11146.091 | params rms: 0.060352 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:08:40.389366 | finish at 2026-10-17 19:19:32 + [2026-10-17 08:14:39] iteration 2202/ 2384 | consumed samples: 70464 | elapsed time per iteration (ms): 227016.0 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.90% | FLOPs estimator: workload-aware | logical tokens/s: 73903.1 | valid tokens/s: 73903.1 | physical tokens/s: 74753.5 | loss tokens/s: 51346.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.453139E-01 | lm-global-supervised-tokens: 1.165642E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.063537E-02 | lm-token-buffer-mib: 2.343187E+02 | global_load_balancing_loss: 1.001352E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11146.053 | params rms: 0.060352 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:28:36.907927 | finish at 2026-10-17 19:43:15 + [2026-10-17 08:18:19] iteration 2203/ 2384 | consumed samples: 70496 | elapsed time per iteration (ms): 220755.7 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.92% | FLOPs estimator: workload-aware | logical tokens/s: 75999.0 | valid tokens/s: 75999.0 | physical tokens/s: 76878.6 | loss tokens/s: 58949.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.118364E-01 | lm-global-supervised-tokens: 1.301345E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.595759E-02 | lm-token-buffer-mib: 2.343263E+02 | global_load_balancing_loss: 9.995444E-01 | loss scale: 1.0 | grad norm: 0.146 | params norm: 11146.017 | params rms: 0.060352 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:05:56.773046 | finish at 2026-10-17 19:24:16 + [2026-10-17 08:21:59] iteration 2204/ 2384 | consumed samples: 70528 | elapsed time per iteration (ms): 219498.4 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.50% | FLOPs estimator: workload-aware | logical tokens/s: 76434.3 | valid tokens/s: 76434.3 | physical tokens/s: 76939.9 | loss tokens/s: 58211.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.645620E-01 | lm-global-supervised-tokens: 1.277742E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.197493E-02 | lm-token-buffer-mib: 2.331721E+02 | global_load_balancing_loss: 1.001683E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11145.982 | params rms: 0.060352 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:58:29.705558 | finish at 2026-10-17 19:20:29 + [2026-10-17 08:25:41] iteration 2205/ 2384 | consumed samples: 70560 | elapsed time per iteration (ms): 222135.5 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.08% | FLOPs estimator: workload-aware | logical tokens/s: 75526.8 | valid tokens/s: 75526.8 | physical tokens/s: 75884.8 | loss tokens/s: 56901.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.625407E-01 | lm-global-supervised-tokens: 1.263989E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.548401E-02 | lm-token-buffer-mib: 2.325375E+02 | global_load_balancing_loss: 1.000191E+00 | loss scale: 1.0 | grad norm: 0.159 | params norm: 11145.947 | params rms: 0.060352 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:02:42.254193 | finish at 2026-10-17 19:28:23 + [2026-10-17 08:29:27] iteration 2206/ 2384 | consumed samples: 70592 | elapsed time per iteration (ms): 225816.4 | throughput per GPU (TFLOP/s/GPU): 25.7 | MFU 8.23% | FLOPs estimator: workload-aware | logical tokens/s: 74295.7 | valid tokens/s: 74295.7 | physical tokens/s: 74524.0 | loss tokens/s: 51439.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.448770E-01 | lm-global-supervised-tokens: 1.161578E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.942748E-02 | lm-token-buffer-mib: 2.322808E+02 | global_load_balancing_loss: 1.000743E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11145.912 | params rms: 0.060351 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:09:55.319718 | finish at 2026-10-17 19:39:22 + [2026-10-17 08:33:11] iteration 2207/ 2384 | consumed samples: 70624 | elapsed time per iteration (ms): 223917.6 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.02% | FLOPs estimator: workload-aware | logical tokens/s: 74925.7 | valid tokens/s: 74925.7 | physical tokens/s: 75706.7 | loss tokens/s: 54734.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.301755E-01 | lm-global-supervised-tokens: 1.225600E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.396422E-02 | lm-token-buffer-mib: 2.352497E+02 | global_load_balancing_loss: 9.995894E-01 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11145.875 | params rms: 0.060351 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:00:33.416704 | finish at 2026-10-17 19:33:44 + [2026-10-17 08:36:54] iteration 2208/ 2384 | consumed samples: 70656 | elapsed time per iteration (ms): 222973.0 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.73% | FLOPs estimator: workload-aware | logical tokens/s: 75243.1 | valid tokens/s: 75243.1 | physical tokens/s: 76020.2 | loss tokens/s: 56927.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.608947E-01 | lm-global-supervised-tokens: 1.269319E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.163054E-02 | lm-token-buffer-mib: 2.339449E+02 | global_load_balancing_loss: 9.995162E-01 | loss scale: 1.0 | grad norm: 0.140 | params norm: 11145.841 | params rms: 0.060351 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:54:03.255898 | finish at 2026-10-17 19:30:57 + [2026-10-17 08:40:37] iteration 2209/ 2384 | consumed samples: 70688 | elapsed time per iteration (ms): 223101.5 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.94% | FLOPs estimator: workload-aware | logical tokens/s: 75199.7 | valid tokens/s: 75199.7 | physical tokens/s: 76024.4 | loss tokens/s: 56718.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.923994E-01 | lm-global-supervised-tokens: 1.265399E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.539825E-02 | lm-token-buffer-mib: 2.361932E+02 | global_load_balancing_loss: 9.998859E-01 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11145.806 | params rms: 0.060351 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:50:42.770725 | finish at 2026-10-17 19:31:20 + [2026-10-17 08:44:42] iteration 2210/ 2384 | consumed samples: 70720 | elapsed time per iteration (ms): 244811.9 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.35% | FLOPs estimator: workload-aware | logical tokens/s: 68530.8 | valid tokens/s: 68530.8 | physical tokens/s: 69183.3 | loss tokens/s: 51523.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.806707E-01 | lm-global-supervised-tokens: 1.261361E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.441792E-02 | lm-token-buffer-mib: 2.348061E+02 | global_load_balancing_loss: 1.000821E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11145.769 | params rms: 0.060351 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:49:57.276971 | finish at 2026-10-17 20:34:39 + [2026-10-17 08:48:23] iteration 2211/ 2384 | consumed samples: 70752 | elapsed time per iteration (ms): 221517.5 | throughput per GPU (TFLOP/s/GPU): 18.8 | MFU 6.02% | FLOPs estimator: workload-aware | logical tokens/s: 75737.5 | valid tokens/s: 75737.5 | physical tokens/s: 76659.0 | loss tokens/s: 59529.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.800322E-01 | lm-global-supervised-tokens: 1.318690E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.222995E-02 | lm-token-buffer-mib: 2.360871E+02 | global_load_balancing_loss: 1.000843E+00 | loss scale: 1.0 | grad norm: 0.155 | params norm: 11145.730 | params rms: 0.060351 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:38:42.522537 | finish at 2026-10-17 19:27:06 + [2026-10-17 08:52:07] iteration 2212/ 2384 | consumed samples: 70784 | elapsed time per iteration (ms): 223594.8 | throughput per GPU (TFLOP/s/GPU): 24.1 | MFU 7.72% | FLOPs estimator: workload-aware | logical tokens/s: 75034.0 | valid tokens/s: 75034.0 | physical tokens/s: 75571.3 | loss tokens/s: 53400.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.011872E-01 | lm-global-supervised-tokens: 1.194016E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.884196E-02 | lm-token-buffer-mib: 2.326820E+02 | global_load_balancing_loss: 1.000976E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11145.696 | params rms: 0.060350 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:40:58.299284 | finish at 2026-10-17 19:33:05 + [2026-10-17 08:55:50] iteration 2213/ 2384 | consumed samples: 70816 | elapsed time per iteration (ms): 222820.1 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.46% | FLOPs estimator: workload-aware | logical tokens/s: 75294.9 | valid tokens/s: 75294.9 | physical tokens/s: 75918.8 | loss tokens/s: 59981.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.778110E-01 | lm-global-supervised-tokens: 1.336497E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.190508E-02 | lm-token-buffer-mib: 2.339304E+02 | global_load_balancing_loss: 1.000618E+00 | loss scale: 1.0 | grad norm: 0.128 | params norm: 11145.664 | params rms: 0.060350 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:35:02.230385 | finish at 2026-10-17 19:30:52 + [2026-10-17 08:59:34] iteration 2214/ 2384 | consumed samples: 70848 | elapsed time per iteration (ms): 224507.6 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.72% | FLOPs estimator: workload-aware | logical tokens/s: 74728.8 | valid tokens/s: 74728.8 | physical tokens/s: 75586.7 | loss tokens/s: 52797.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.578797E-01 | lm-global-supervised-tokens: 1.185333E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.513461E-02 | lm-token-buffer-mib: 2.327751E+02 | global_load_balancing_loss: 1.000816E+00 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11145.633 | params rms: 0.060350 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:36:06.296389 | finish at 2026-10-17 19:35:40 + [2026-10-17 09:03:18] iteration 2215/ 2384 | consumed samples: 70880 | elapsed time per iteration (ms): 224318.6 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.65% | FLOPs estimator: workload-aware | logical tokens/s: 74791.8 | valid tokens/s: 74791.8 | physical tokens/s: 76033.6 | loss tokens/s: 53847.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.163101E-01 | lm-global-supervised-tokens: 1.207902E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.176419E-02 | lm-token-buffer-mib: 2.342715E+02 | global_load_balancing_loss: 1.000369E+00 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11145.600 | params rms: 0.060350 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:31:49.846774 | finish at 2026-10-17 19:35:08 + [2026-10-17 09:07:02] iteration 2216/ 2384 | consumed samples: 70912 | elapsed time per iteration (ms): 223751.3 | throughput per GPU (TFLOP/s/GPU): 23.2 | MFU 7.42% | FLOPs estimator: workload-aware | logical tokens/s: 74981.2 | valid tokens/s: 74981.2 | physical tokens/s: 75616.5 | loss tokens/s: 56555.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.056816E-01 | lm-global-supervised-tokens: 1.265437E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.127808E-02 | lm-token-buffer-mib: 2.342826E+02 | global_load_balancing_loss: 9.997187E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11145.566 | params rms: 0.060350 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:26:30.211967 | finish at 2026-10-17 19:33:32 + [2026-10-17 09:10:42] iteration 2217/ 2384 | consumed samples: 70944 | elapsed time per iteration (ms): 220403.7 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.34% | FLOPs estimator: workload-aware | logical tokens/s: 76120.4 | valid tokens/s: 76120.4 | physical tokens/s: 76462.2 | loss tokens/s: 59200.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.662326E-01 | lm-global-supervised-tokens: 1.304790E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.381846E-02 | lm-token-buffer-mib: 2.330317E+02 | global_load_balancing_loss: 9.990876E-01 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11145.532 | params rms: 0.060349 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:13:27.414614 | finish at 2026-10-17 19:24:10 + [2026-10-17 09:14:24] iteration 2218/ 2384 | consumed samples: 70976 | elapsed time per iteration (ms): 221791.4 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.61% | FLOPs estimator: workload-aware | logical tokens/s: 75644.0 | valid tokens/s: 75644.0 | physical tokens/s: 76471.1 | loss tokens/s: 55501.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.906330E-01 | lm-global-supervised-tokens: 1.230981E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.631985E-02 | lm-token-buffer-mib: 2.329541E+02 | global_load_balancing_loss: 1.000857E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11145.501 | params rms: 0.060349 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:13:37.369781 | finish at 2026-10-17 19:28:02 + [2026-10-17 09:18:07] iteration 2219/ 2384 | consumed samples: 71008 | elapsed time per iteration (ms): 223125.5 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.78% | FLOPs estimator: workload-aware | logical tokens/s: 75191.8 | valid tokens/s: 75191.8 | physical tokens/s: 76110.3 | loss tokens/s: 59783.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.885454E-01 | lm-global-supervised-tokens: 1.333924E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.330875E-02 | lm-token-buffer-mib: 2.328937E+02 | global_load_balancing_loss: 1.000396E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11145.470 | params rms: 0.060349 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:13:35.700846 | finish at 2026-10-17 19:31:43 + [2026-10-17 09:22:09] iteration 2220/ 2384 | consumed samples: 71040 | elapsed time per iteration (ms): 241913.1 | throughput per GPU (TFLOP/s/GPU): 18.1 | MFU 5.81% | FLOPs estimator: workload-aware | logical tokens/s: 69352.1 | valid tokens/s: 69352.1 | physical tokens/s: 70049.1 | loss tokens/s: 53209.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.611268E-01 | lm-global-supervised-tokens: 1.287217E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.593847E-02 | lm-token-buffer-mib: 2.344863E+02 | global_load_balancing_loss: 1.001280E+00 | loss scale: 1.0 | grad norm: 0.131 | params norm: 11145.438 | params rms: 0.060349 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:01:13.745859 | finish at 2026-10-17 20:23:23 + [2026-10-17 09:25:54] iteration 2221/ 2384 | consumed samples: 71072 | elapsed time per iteration (ms): 224847.5 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.32% | FLOPs estimator: workload-aware | logical tokens/s: 74615.9 | valid tokens/s: 74615.9 | physical tokens/s: 74910.9 | loss tokens/s: 55239.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.257086E-01 | lm-global-supervised-tokens: 1.242041E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.245752E-02 | lm-token-buffer-mib: 2.324553E+02 | global_load_balancing_loss: 1.001454E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11145.406 | params rms: 0.060349 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:10:50.140687 | finish at 2026-10-17 19:36:44 + [2026-10-17 09:29:36] iteration 2222/ 2384 | consumed samples: 71104 | elapsed time per iteration (ms): 221530.7 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.46% | FLOPs estimator: workload-aware | logical tokens/s: 75733.1 | valid tokens/s: 75733.1 | physical tokens/s: 76523.6 | loss tokens/s: 54724.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.148356E-01 | lm-global-supervised-tokens: 1.212306E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.164497E-02 | lm-token-buffer-mib: 2.333749E+02 | global_load_balancing_loss: 9.990492E-01 | loss scale: 1.0 | grad norm: 0.352 | params norm: 11145.374 | params rms: 0.060349 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:58:07.975751 | finish at 2026-10-17 19:27:44 + [2026-10-17 09:33:21] iteration 2223/ 2384 | consumed samples: 71136 | elapsed time per iteration (ms): 225204.8 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.01% | FLOPs estimator: workload-aware | logical tokens/s: 74497.6 | valid tokens/s: 74497.6 | physical tokens/s: 76344.5 | loss tokens/s: 56847.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.541861E-01 | lm-global-supervised-tokens: 1.280237E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.177015E-02 | lm-token-buffer-mib: 2.374410E+02 | global_load_balancing_loss: 9.997643E-01 | loss scale: 1.0 | grad norm: 0.127 | params norm: 11145.339 | params rms: 0.060348 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:04:17.979154 | finish at 2026-10-17 19:37:39 + [2026-10-17 09:37:01] iteration 2224/ 2384 | consumed samples: 71168 | elapsed time per iteration (ms): 220400.5 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.69% | FLOPs estimator: workload-aware | logical tokens/s: 76121.3 | valid tokens/s: 76121.3 | physical tokens/s: 76713.0 | loss tokens/s: 59029.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.920615E-01 | lm-global-supervised-tokens: 1.301007E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.334961E-02 | lm-token-buffer-mib: 2.328692E+02 | global_load_balancing_loss: 1.000455E+00 | loss scale: 1.0 | grad norm: 0.187 | params norm: 11145.307 | params rms: 0.060348 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:47:44.087868 | finish at 2026-10-17 19:24:45 + [2026-10-17 09:40:46] iteration 2225/ 2384 | consumed samples: 71200 | elapsed time per iteration (ms): 224545.8 | throughput per GPU (TFLOP/s/GPU): 23.1 | MFU 7.40% | FLOPs estimator: workload-aware | logical tokens/s: 74716.1 | valid tokens/s: 74716.1 | physical tokens/s: 75560.2 | loss tokens/s: 58867.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.053131E-01 | lm-global-supervised-tokens: 1.321836E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.177156E-02 | lm-token-buffer-mib: 2.344593E+02 | global_load_balancing_loss: 9.999942E-01 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11145.274 | params rms: 0.060348 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:55:02.783105 | finish at 2026-10-17 19:35:49 + [2026-10-17 09:44:27] iteration 2226/ 2384 | consumed samples: 71232 | elapsed time per iteration (ms): 221575.0 | throughput per GPU (TFLOP/s/GPU): 18.8 | MFU 6.04% | FLOPs estimator: workload-aware | logical tokens/s: 75718.0 | valid tokens/s: 75718.0 | physical tokens/s: 76510.0 | loss tokens/s: 60308.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.465597E-01 | lm-global-supervised-tokens: 1.336284E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.980301E-02 | lm-token-buffer-mib: 2.355543E+02 | global_load_balancing_loss: 1.000370E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11145.236 | params rms: 0.060348 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:43:28.849103 | finish at 2026-10-17 19:27:56 + [2026-10-17 09:48:15] iteration 2227/ 2384 | consumed samples: 71264 | elapsed time per iteration (ms): 227121.0 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.60% | FLOPs estimator: workload-aware | logical tokens/s: 73869.0 | valid tokens/s: 73868.9 | physical tokens/s: 75033.7 | loss tokens/s: 55437.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.426036E-01 | lm-global-supervised-tokens: 1.259090E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.410203E-02 | lm-token-buffer-mib: 2.355953E+02 | global_load_balancing_loss: 1.000809E+00 | loss scale: 1.0 | grad norm: 0.220 | params norm: 11145.201 | params rms: 0.060348 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:54:17.997457 | finish at 2026-10-17 19:42:33 + [2026-10-17 09:51:57] iteration 2228/ 2384 | consumed samples: 71296 | elapsed time per iteration (ms): 222639.6 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.47% | FLOPs estimator: workload-aware | logical tokens/s: 75355.8 | valid tokens/s: 75355.8 | physical tokens/s: 75540.5 | loss tokens/s: 51657.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.871469E-01 | lm-global-supervised-tokens: 1.150111E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.573762E-02 | lm-token-buffer-mib: 2.327422E+02 | global_load_balancing_loss: 1.000307E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11145.166 | params rms: 0.060347 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:38:51.783966 | finish at 2026-10-17 19:30:49 + [2026-10-17 09:55:41] iteration 2229/ 2384 | consumed samples: 71328 | elapsed time per iteration (ms): 223706.1 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.97% | FLOPs estimator: workload-aware | logical tokens/s: 74996.6 | valid tokens/s: 74996.6 | physical tokens/s: 76489.5 | loss tokens/s: 61197.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.178007E-01 | lm-global-supervised-tokens: 1.369033E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.017903E-02 | lm-token-buffer-mib: 2.350562E+02 | global_load_balancing_loss: 1.001116E+00 | loss scale: 1.0 | grad norm: 0.303 | params norm: 11145.133 | params rms: 0.060347 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:37:54.449933 | finish at 2026-10-17 19:33:35 + [2026-10-17 09:59:46] iteration 2230/ 2384 | consumed samples: 71360 | elapsed time per iteration (ms): 244860.3 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.66% | FLOPs estimator: workload-aware | logical tokens/s: 68517.5 | valid tokens/s: 68517.5 | physical tokens/s: 68762.9 | loss tokens/s: 50134.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.822709E-01 | lm-global-supervised-tokens: 1.227592E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.377819E-02 | lm-token-buffer-mib: 2.324221E+02 | global_load_balancing_loss: 1.000670E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11145.102 | params rms: 0.060347 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:28:28.489771 | finish at 2026-10-17 20:28:14 + [2026-10-17 10:03:29] iteration 2231/ 2384 | consumed samples: 71392 | elapsed time per iteration (ms): 222882.6 | throughput per GPU (TFLOP/s/GPU): 18.9 | MFU 6.07% | FLOPs estimator: workload-aware | logical tokens/s: 75273.8 | valid tokens/s: 75273.8 | physical tokens/s: 76205.0 | loss tokens/s: 56896.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.478009E-01 | lm-global-supervised-tokens: 1.268132E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.422490E-02 | lm-token-buffer-mib: 2.370168E+02 | global_load_balancing_loss: 9.995787E-01 | loss scale: 1.0 | grad norm: 0.156 | params norm: 11145.068 | params rms: 0.060347 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:28:21.037373 | finish at 2026-10-17 19:31:50 + [2026-10-17 10:07:11] iteration 2232/ 2384 | consumed samples: 71424 | elapsed time per iteration (ms): 222109.1 | throughput per GPU (TFLOP/s/GPU): 19.3 | MFU 6.18% | FLOPs estimator: workload-aware | logical tokens/s: 75535.8 | valid tokens/s: 75535.8 | physical tokens/s: 76376.5 | loss tokens/s: 56947.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.816879E-01 | lm-global-supervised-tokens: 1.264857E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.756125E-02 | lm-token-buffer-mib: 2.350882E+02 | global_load_balancing_loss: 9.988151E-01 | loss scale: 1.0 | grad norm: 0.157 | params norm: 11145.035 | params rms: 0.060347 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:22:40.577526 | finish at 2026-10-17 19:29:51 + [2026-10-17 10:10:52] iteration 2233/ 2384 | consumed samples: 71456 | elapsed time per iteration (ms): 221496.7 | throughput per GPU (TFLOP/s/GPU): 22.0 | MFU 7.05% | FLOPs estimator: workload-aware | logical tokens/s: 75744.7 | valid tokens/s: 75744.7 | physical tokens/s: 76794.5 | loss tokens/s: 58793.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.372020E-01 | lm-global-supervised-tokens: 1.302245E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 6.317694E-02 | lm-token-buffer-mib: 2.368417E+02 | global_load_balancing_loss: 9.996849E-01 | loss scale: 1.0 | grad norm: 0.379 | params norm: 11145.000 | params rms: 0.060347 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:17:26.002139 | finish at 2026-10-17 19:28:18 + [2026-10-17 10:14:36] iteration 2234/ 2384 | consumed samples: 71488 | elapsed time per iteration (ms): 223572.1 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.37% | FLOPs estimator: workload-aware | logical tokens/s: 75041.5 | valid tokens/s: 75041.5 | physical tokens/s: 76001.5 | loss tokens/s: 53821.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.029502E-01 | lm-global-supervised-tokens: 1.203288E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.153107E-02 | lm-token-buffer-mib: 2.356850E+02 | global_load_balancing_loss: 1.000738E+00 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11144.967 | params rms: 0.060346 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:18:55.821140 | finish at 2026-10-17 19:33:32 + [2026-10-17 10:18:21] iteration 2235/ 2384 | consumed samples: 71520 | elapsed time per iteration (ms): 225417.3 | throughput per GPU (TFLOP/s/GPU): 23.3 | MFU 7.46% | FLOPs estimator: workload-aware | logical tokens/s: 74427.3 | valid tokens/s: 74427.3 | physical tokens/s: 74689.7 | loss tokens/s: 53382.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.180833E-01 | lm-global-supervised-tokens: 1.203334E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.777310E-02 | lm-token-buffer-mib: 2.332969E+02 | global_load_balancing_loss: 1.000137E+00 | loss scale: 1.0 | grad norm: 0.303 | params norm: 11144.935 | params rms: 0.060346 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:19:47.177769 | finish at 2026-10-17 19:38:08 + [2026-10-17 10:22:03] iteration 2236/ 2384 | consumed samples: 71552 | elapsed time per iteration (ms): 221499.1 | throughput per GPU (TFLOP/s/GPU): 18.9 | MFU 6.05% | FLOPs estimator: workload-aware | logical tokens/s: 75743.9 | valid tokens/s: 75743.9 | physical tokens/s: 76396.9 | loss tokens/s: 57369.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.636746E-01 | lm-global-supervised-tokens: 1.270720E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.266204E-02 | lm-token-buffer-mib: 2.341833E+02 | global_load_balancing_loss: 1.000844E+00 | loss scale: 1.0 | grad norm: 0.117 | params norm: 11144.908 | params rms: 0.060346 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:06:21.869477 | finish at 2026-10-17 19:28:25 + [2026-10-17 10:25:44] iteration 2237/ 2384 | consumed samples: 71584 | elapsed time per iteration (ms): 221583.0 | throughput per GPU (TFLOP/s/GPU): 18.8 | MFU 6.03% | FLOPs estimator: workload-aware | logical tokens/s: 75715.3 | valid tokens/s: 75715.3 | physical tokens/s: 76773.0 | loss tokens/s: 60204.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.798818E-01 | lm-global-supervised-tokens: 1.334035E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.231219E-02 | lm-token-buffer-mib: 2.362738E+02 | global_load_balancing_loss: 1.001072E+00 | loss scale: 1.0 | grad norm: 0.349 | params norm: 11144.882 | params rms: 0.060346 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:02:52.695595 | finish at 2026-10-17 19:28:37 + [2026-10-17 10:29:27] iteration 2238/ 2384 | consumed samples: 71616 | elapsed time per iteration (ms): 222973.1 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.41% | FLOPs estimator: workload-aware | logical tokens/s: 75243.2 | valid tokens/s: 75243.2 | physical tokens/s: 76099.2 | loss tokens/s: 59857.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.025834E-01 | lm-global-supervised-tokens: 1.334655E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.860772E-02 | lm-token-buffer-mib: 2.355368E+02 | global_load_balancing_loss: 1.001524E+00 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11144.856 | params rms: 0.060346 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:02:34.068485 | finish at 2026-10-17 19:32:01 + [2026-10-17 10:33:09] iteration 2239/ 2384 | consumed samples: 71648 | elapsed time per iteration (ms): 221476.8 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.96% | FLOPs estimator: workload-aware | logical tokens/s: 75751.4 | valid tokens/s: 75751.4 | physical tokens/s: 76754.3 | loss tokens/s: 60777.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.661242E-01 | lm-global-supervised-tokens: 1.346074E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.461720E-02 | lm-token-buffer-mib: 2.338425E+02 | global_load_balancing_loss: 9.996938E-01 | loss scale: 1.0 | grad norm: 0.335 | params norm: 11144.828 | params rms: 0.060346 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:55:14.142460 | finish at 2026-10-17 19:28:23 + [2026-10-17 10:37:12] iteration 2240/ 2384 | consumed samples: 71680 | elapsed time per iteration (ms): 243223.4 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.44% | FLOPs estimator: workload-aware | logical tokens/s: 68978.5 | valid tokens/s: 68978.5 | physical tokens/s: 69474.1 | loss tokens/s: 51507.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.011317E-01 | lm-global-supervised-tokens: 1.252773E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.204164E-02 | lm-token-buffer-mib: 2.340427E+02 | global_load_balancing_loss: 1.000769E+00 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11144.798 | params rms: 0.060345 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:43:44.163712 | finish at 2026-10-17 20:20:56 +times across ranks (ms): + save-checkpoint: + rank 0: 24911.72 + rank 1: 24911.73 + rank 2: 24911.74 + rank 3: 24911.73 + rank 4: 24911.75 + rank 5: 24911.75 + rank 6: 24911.75 + rank 7: 24911.73 + rank 8: 24911.98 + rank 9: 24912.00 + rank 10: 24911.85 + rank 11: 24912.00 + rank 12: 24911.97 + rank 13: 24911.97 + rank 14: 24911.99 + rank 15: 24911.99 + rank 16: 24911.45 + rank 17: 24911.43 + rank 18: 24911.45 + rank 19: 24911.43 + rank 20: 24911.42 + rank 21: 24911.42 + rank 22: 24911.44 + rank 23: 24911.41 + rank 24: 24912.11 + rank 25: 24912.23 + rank 26: 24912.11 + rank 27: 24912.18 + rank 28: 24912.89 + rank 29: 24912.21 + rank 30: 24912.18 + rank 31: 24912.21 + rank 32: 24912.59 + rank 33: 24912.39 + rank 34: 24912.53 + rank 35: 24912.57 + rank 36: 24912.46 + rank 37: 24912.53 + rank 38: 24912.44 + rank 39: 24912.37 + rank 40: 24911.39 + rank 41: 24911.52 + rank 42: 24911.52 + rank 43: 24911.54 + rank 44: 24911.52 + rank 45: 24911.52 + rank 46: 24911.54 + rank 47: 24911.52 + rank 48: 24912.04 + rank 49: 24911.93 + rank 50: 24912.07 + rank 51: 24912.03 + rank 52: 24912.06 + rank 53: 24912.09 + rank 54: 24912.00 + rank 55: 24912.03 + rank 56: 24912.16 + rank 57: 24912.19 + rank 58: 24912.24 + rank 59: 24912.19 + rank 60: 24912.18 + rank 61: 24912.15 + rank 62: 24912.15 + rank 63: 24912.15 + [2026-10-17 10:41:19] iteration 2241/ 2384 | consumed samples: 71712 | elapsed time per iteration (ms): 221477.9 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.85% | FLOPs estimator: workload-aware | logical tokens/s: 75751.2 | valid tokens/s: 75751.2 | physical tokens/s: 76368.2 | loss tokens/s: 55797.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.758264E-01 | lm-global-supervised-tokens: 1.235796E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.532024E-01 | lm-token-buffer-mib: 2.340783E+02 | global_load_balancing_loss: 1.001272E+00 | loss scale: 1.0 | grad norm: 0.116 | params norm: 11144.766 | params rms: 0.060345 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:47:51.336226 | finish at 2026-10-17 19:29:10 + [2026-10-17 10:45:06] iteration 2242/ 2384 | consumed samples: 71744 | elapsed time per iteration (ms): 227430.2 | throughput per GPU (TFLOP/s/GPU): 23.8 | MFU 7.64% | FLOPs estimator: workload-aware | logical tokens/s: 73768.4 | valid tokens/s: 73768.4 | physical tokens/s: 74562.5 | loss tokens/s: 54871.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.365123E-01 | lm-global-supervised-tokens: 1.247941E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.230253E-02 | lm-token-buffer-mib: 2.338152E+02 | global_load_balancing_loss: 1.001401E+00 | loss scale: 1.0 | grad norm: 0.267 | params norm: 11144.735 | params rms: 0.060345 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:58:15.085063 | finish at 2026-10-17 19:43:21 + [2026-10-17 10:48:47] iteration 2243/ 2384 | consumed samples: 71776 | elapsed time per iteration (ms): 220467.3 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.73% | FLOPs estimator: workload-aware | logical tokens/s: 76098.2 | valid tokens/s: 76098.2 | physical tokens/s: 76773.9 | loss tokens/s: 56035.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.895812E-01 | lm-global-supervised-tokens: 1.235409E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.293574E-02 | lm-token-buffer-mib: 2.339805E+02 | global_load_balancing_loss: 9.999702E-01 | loss scale: 1.0 | grad norm: 0.157 | params norm: 11144.704 | params rms: 0.060345 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:38:05.895645 | finish at 2026-10-17 19:26:52 + [2026-10-17 10:52:27] iteration 2244/ 2384 | consumed samples: 71808 | elapsed time per iteration (ms): 220791.4 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.29% | FLOPs estimator: workload-aware | logical tokens/s: 75986.6 | valid tokens/s: 75986.6 | physical tokens/s: 76667.3 | loss tokens/s: 55119.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.544624E-01 | lm-global-supervised-tokens: 1.216993E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.325005E-02 | lm-token-buffer-mib: 2.354894E+02 | global_load_balancing_loss: 9.990639E-01 | loss scale: 1.0 | grad norm: 0.103 | params norm: 11144.677 | params rms: 0.060345 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:35:10.798631 | finish at 2026-10-17 19:27:38 + [2026-10-17 10:56:08] iteration 2245/ 2384 | consumed samples: 71840 | elapsed time per iteration (ms): 220467.7 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.90% | FLOPs estimator: workload-aware | logical tokens/s: 76098.1 | valid tokens/s: 76098.0 | physical tokens/s: 76497.4 | loss tokens/s: 57062.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.083470E-01 | lm-global-supervised-tokens: 1.258033E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.432294E-02 | lm-token-buffer-mib: 2.336987E+02 | global_load_balancing_loss: 9.998074E-01 | loss scale: 1.0 | grad norm: 0.154 | params norm: 11144.651 | params rms: 0.060345 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:30:45.009008 | finish at 2026-10-17 19:26:53 + [2026-10-17 10:59:48] iteration 2246/ 2384 | consumed samples: 71872 | elapsed time per iteration (ms): 219668.9 | throughput per GPU (TFLOP/s/GPU): 23.6 | MFU 7.56% | FLOPs estimator: workload-aware | logical tokens/s: 76374.9 | valid tokens/s: 76374.9 | physical tokens/s: 76947.5 | loss tokens/s: 55022.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.124114E-01 | lm-global-supervised-tokens: 1.208667E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.276779E-02 | lm-token-buffer-mib: 2.341283E+02 | global_load_balancing_loss: 9.991779E-01 | loss scale: 1.0 | grad norm: 0.231 | params norm: 11144.623 | params rms: 0.060345 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:25:14.309584 | finish at 2026-10-17 19:25:02 + [2026-10-17 11:03:27] iteration 2247/ 2384 | consumed samples: 71904 | elapsed time per iteration (ms): 219915.2 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.03% | FLOPs estimator: workload-aware | logical tokens/s: 76289.4 | valid tokens/s: 76289.4 | physical tokens/s: 76956.8 | loss tokens/s: 59158.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.469564E-01 | lm-global-supervised-tokens: 1.300981E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.681125E-02 | lm-token-buffer-mib: 2.337374E+02 | global_load_balancing_loss: 9.996982E-01 | loss scale: 1.0 | grad norm: 0.169 | params norm: 11144.595 | params rms: 0.060344 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:22:08.379884 | finish at 2026-10-17 19:25:36 + [2026-10-17 11:07:09] iteration 2248/ 2384 | consumed samples: 71936 | elapsed time per iteration (ms): 222037.7 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.13% | FLOPs estimator: workload-aware | logical tokens/s: 75560.1 | valid tokens/s: 75560.1 | physical tokens/s: 76319.4 | loss tokens/s: 58996.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.437628E-01 | lm-global-supervised-tokens: 1.309934E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.457303E-02 | lm-token-buffer-mib: 2.348109E+02 | global_load_balancing_loss: 1.000916E+00 | loss scale: 1.0 | grad norm: 0.209 | params norm: 11144.565 | params rms: 0.060344 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:23:17.133644 | finish at 2026-10-17 19:30:27 + [2026-10-17 11:10:57] iteration 2249/ 2384 | consumed samples: 71968 | elapsed time per iteration (ms): 227941.4 | throughput per GPU (TFLOP/s/GPU): 19.9 | MFU 6.37% | FLOPs estimator: workload-aware | logical tokens/s: 73603.2 | valid tokens/s: 73603.2 | physical tokens/s: 75480.7 | loss tokens/s: 56011.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.450254E-01 | lm-global-supervised-tokens: 1.276741E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.421192E-02 | lm-token-buffer-mib: 2.397916E+02 | global_load_balancing_loss: 1.000165E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11144.538 | params rms: 0.060344 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:32:52.085305 | finish at 2026-10-17 19:43:49 + [2026-10-17 11:15:02] iteration 2250/ 2384 | consumed samples: 72000 | elapsed time per iteration (ms): 244967.3 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.10% | FLOPs estimator: workload-aware | logical tokens/s: 68487.5 | valid tokens/s: 68487.5 | physical tokens/s: 68957.8 | loss tokens/s: 51839.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.011516E-01 | lm-global-supervised-tokens: 1.269907E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.127230E-02 | lm-token-buffer-mib: 2.340351E+02 | global_load_balancing_loss: 1.000745E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11144.511 | params rms: 0.060344 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:07:05.624805 | finish at 2026-10-17 20:22:08 + [2026-10-17 11:18:46] iteration 2251/ 2384 | consumed samples: 72032 | elapsed time per iteration (ms): 223344.9 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.14% | FLOPs estimator: workload-aware | logical tokens/s: 75118.0 | valid tokens/s: 75118.0 | physical tokens/s: 75973.3 | loss tokens/s: 56318.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.820056E-01 | lm-global-supervised-tokens: 1.257849E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.511421E-02 | lm-token-buffer-mib: 2.339058E+02 | global_load_balancing_loss: 9.995587E-01 | loss scale: 1.0 | grad norm: 0.305 | params norm: 11144.483 | params rms: 0.060344 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:15:04.869244 | finish at 2026-10-17 19:33:51 + [2026-10-17 11:22:29] iteration 2252/ 2384 | consumed samples: 72064 | elapsed time per iteration (ms): 223588.3 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.81% | FLOPs estimator: workload-aware | logical tokens/s: 75036.0 | valid tokens/s: 75036.0 | physical tokens/s: 75305.0 | loss tokens/s: 56610.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.558161E-01 | lm-global-supervised-tokens: 1.265742E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.194622E-02 | lm-token-buffer-mib: 2.326998E+02 | global_load_balancing_loss: 1.001279E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11144.456 | params rms: 0.060344 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:11:53.661295 | finish at 2026-10-17 19:34:23 + [2026-10-17 11:26:11] iteration 2253/ 2384 | consumed samples: 72096 | elapsed time per iteration (ms): 221251.7 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.25% | FLOPs estimator: workload-aware | logical tokens/s: 75828.6 | valid tokens/s: 75828.6 | physical tokens/s: 76812.1 | loss tokens/s: 57995.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.799558E-01 | lm-global-supervised-tokens: 1.283149E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.951302E-02 | lm-token-buffer-mib: 2.326909E+02 | global_load_balancing_loss: 1.000101E+00 | loss scale: 1.0 | grad norm: 0.344 | params norm: 11144.429 | params rms: 0.060343 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:03:03.970629 | finish at 2026-10-17 19:29:15 + [2026-10-17 11:29:52] iteration 2254/ 2384 | consumed samples: 72128 | elapsed time per iteration (ms): 221554.7 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.83% | FLOPs estimator: workload-aware | logical tokens/s: 75724.9 | valid tokens/s: 75724.9 | physical tokens/s: 76719.8 | loss tokens/s: 59147.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.500040E-01 | lm-global-supervised-tokens: 1.310430E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 5.914534E-02 | lm-token-buffer-mib: 2.326736E+02 | global_load_balancing_loss: 1.000081E+00 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11144.403 | params rms: 0.060343 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:00:02.114551 | finish at 2026-10-17 19:29:54 + [2026-10-17 11:33:35] iteration 2255/ 2384 | consumed samples: 72160 | elapsed time per iteration (ms): 222391.6 | throughput per GPU (TFLOP/s/GPU): 19.7 | MFU 6.30% | FLOPs estimator: workload-aware | logical tokens/s: 75439.8 | valid tokens/s: 75439.8 | physical tokens/s: 76277.1 | loss tokens/s: 55289.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.149273E-01 | lm-global-supervised-tokens: 1.229580E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.477967E-02 | lm-token-buffer-mib: 2.357766E+02 | global_load_balancing_loss: 1.001442E+00 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11144.376 | params rms: 0.060343 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:58:08.511312 | finish at 2026-10-17 19:31:43 + [2026-10-17 11:37:18] iteration 2256/ 2384 | consumed samples: 72192 | elapsed time per iteration (ms): 223677.4 | throughput per GPU (TFLOP/s/GPU): 25.6 | MFU 8.20% | FLOPs estimator: workload-aware | logical tokens/s: 75006.2 | valid tokens/s: 75006.2 | physical tokens/s: 76268.4 | loss tokens/s: 58928.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.811281E-01 | lm-global-supervised-tokens: 1.318092E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.863487E-02 | lm-token-buffer-mib: 2.335202E+02 | global_load_balancing_loss: 1.000221E+00 | loss scale: 1.0 | grad norm: 0.165 | params norm: 11144.349 | params rms: 0.060343 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:57:10.713379 | finish at 2026-10-17 19:34:29 + [2026-10-17 11:41:02] iteration 2257/ 2384 | consumed samples: 72224 | elapsed time per iteration (ms): 223551.8 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.03% | FLOPs estimator: workload-aware | logical tokens/s: 75048.4 | valid tokens/s: 75048.4 | physical tokens/s: 75612.1 | loss tokens/s: 57454.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.846853E-01 | lm-global-supervised-tokens: 1.284397E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.059609E-02 | lm-token-buffer-mib: 2.341070E+02 | global_load_balancing_loss: 1.000856E+00 | loss scale: 1.0 | grad norm: 0.123 | params norm: 11144.321 | params rms: 0.060343 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:53:11.081236 | finish at 2026-10-17 19:34:13 + [2026-10-17 11:44:45] iteration 2258/ 2384 | consumed samples: 72256 | elapsed time per iteration (ms): 223288.8 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.31% | FLOPs estimator: workload-aware | logical tokens/s: 75136.7 | valid tokens/s: 75136.7 | physical tokens/s: 76000.7 | loss tokens/s: 58394.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.816934E-01 | lm-global-supervised-tokens: 1.303877E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.349487E-02 | lm-token-buffer-mib: 2.351922E+02 | global_load_balancing_loss: 9.996940E-01 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11144.294 | params rms: 0.060343 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:48:54.390603 | finish at 2026-10-17 19:33:39 + [2026-10-17 11:48:25] iteration 2259/ 2384 | consumed samples: 72288 | elapsed time per iteration (ms): 219679.9 | throughput per GPU (TFLOP/s/GPU): 17.6 | MFU 5.65% | FLOPs estimator: workload-aware | logical tokens/s: 76371.2 | valid tokens/s: 76371.2 | physical tokens/s: 77226.0 | loss tokens/s: 62187.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.593972E-01 | lm-global-supervised-tokens: 1.366132E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.892424E-02 | lm-token-buffer-mib: 2.346969E+02 | global_load_balancing_loss: 9.991267E-01 | loss scale: 1.0 | grad norm: 0.151 | params norm: 11144.270 | params rms: 0.060343 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:37:39.982336 | finish at 2026-10-17 19:26:05 + [2026-10-17 11:52:27] iteration 2260/ 2384 | consumed samples: 72320 | elapsed time per iteration (ms): 242035.5 | throughput per GPU (TFLOP/s/GPU): 19.7 | MFU 6.32% | FLOPs estimator: workload-aware | logical tokens/s: 69317.0 | valid tokens/s: 69317.0 | physical tokens/s: 69686.1 | loss tokens/s: 53290.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.576082E-01 | lm-global-supervised-tokens: 1.289818E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.152134E-02 | lm-token-buffer-mib: 2.338270E+02 | global_load_balancing_loss: 9.999552E-01 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11144.244 | params rms: 0.060342 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:20:12.396152 | finish at 2026-10-17 20:12:39 + [2026-10-17 11:56:08] iteration 2261/ 2384 | consumed samples: 72352 | elapsed time per iteration (ms): 221690.8 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.14% | FLOPs estimator: workload-aware | logical tokens/s: 75678.3 | valid tokens/s: 75678.3 | physical tokens/s: 77137.8 | loss tokens/s: 58270.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.317357E-01 | lm-global-supervised-tokens: 1.291803E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.738289E-02 | lm-token-buffer-mib: 2.346518E+02 | global_load_balancing_loss: 9.998119E-01 | loss scale: 1.0 | grad norm: 0.112 | params norm: 11144.218 | params rms: 0.060342 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:34:27.972148 | finish at 2026-10-17 19:30:36 + [2026-10-17 11:59:54] iteration 2262/ 2384 | consumed samples: 72384 | elapsed time per iteration (ms): 226027.1 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.70% | FLOPs estimator: workload-aware | logical tokens/s: 74226.4 | valid tokens/s: 74226.3 | physical tokens/s: 75201.2 | loss tokens/s: 57901.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.674716E-01 | lm-global-supervised-tokens: 1.308728E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.868023E-02 | lm-token-buffer-mib: 2.369662E+02 | global_load_balancing_loss: 1.001412E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11144.190 | params rms: 0.060342 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:39:35.307403 | finish at 2026-10-17 19:39:30 + [2026-10-17 12:03:40] iteration 2263/ 2384 | consumed samples: 72416 | elapsed time per iteration (ms): 225344.7 | throughput per GPU (TFLOP/s/GPU): 24.9 | MFU 7.98% | FLOPs estimator: workload-aware | logical tokens/s: 74451.3 | valid tokens/s: 74451.3 | physical tokens/s: 74971.9 | loss tokens/s: 56783.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.338236E-01 | lm-global-supervised-tokens: 1.279575E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.193785E-02 | lm-token-buffer-mib: 2.330412E+02 | global_load_balancing_loss: 1.000914E+00 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11144.164 | params rms: 0.060342 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:34:26.713876 | finish at 2026-10-17 19:38:07 + [2026-10-17 12:07:21] iteration 2264/ 2384 | consumed samples: 72448 | elapsed time per iteration (ms): 220721.1 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.50% | FLOPs estimator: workload-aware | logical tokens/s: 76010.7 | valid tokens/s: 76010.7 | physical tokens/s: 76674.1 | loss tokens/s: 53657.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.912462E-01 | lm-global-supervised-tokens: 1.184342E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.350939E-02 | lm-token-buffer-mib: 2.340166E+02 | global_load_balancing_loss: 1.001709E+00 | loss scale: 1.0 | grad norm: 0.179 | params norm: 11144.139 | params rms: 0.060342 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:21:26.535215 | finish at 2026-10-17 19:28:47 + [2026-10-17 12:11:05] iteration 2265/ 2384 | consumed samples: 72480 | elapsed time per iteration (ms): 224802.6 | throughput per GPU (TFLOP/s/GPU): 21.2 | MFU 6.78% | FLOPs estimator: workload-aware | logical tokens/s: 74630.8 | valid tokens/s: 74630.8 | physical tokens/s: 75712.7 | loss tokens/s: 59135.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.975692E-01 | lm-global-supervised-tokens: 1.329377E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.497095E-02 | lm-token-buffer-mib: 2.346071E+02 | global_load_balancing_loss: 1.000247E+00 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11144.116 | params rms: 0.060342 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:25:51.512340 | finish at 2026-10-17 19:36:57 + [2026-10-17 12:14:47] iteration 2266/ 2384 | consumed samples: 72512 | elapsed time per iteration (ms): 221264.5 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.90% | FLOPs estimator: workload-aware | logical tokens/s: 75824.1 | valid tokens/s: 75824.0 | physical tokens/s: 76949.7 | loss tokens/s: 50198.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.841061E-01 | lm-global-supervised-tokens: 1.110717E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.263110E-02 | lm-token-buffer-mib: 2.324573E+02 | global_load_balancing_loss: 9.990722E-01 | loss scale: 1.0 | grad norm: 0.104 | params norm: 11144.093 | params rms: 0.060342 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:15:09.207022 | finish at 2026-10-17 19:29:56 + [2026-10-17 12:18:30] iteration 2267/ 2384 | consumed samples: 72544 | elapsed time per iteration (ms): 222945.7 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.83% | FLOPs estimator: workload-aware | logical tokens/s: 75252.4 | valid tokens/s: 75252.4 | physical tokens/s: 75779.5 | loss tokens/s: 56429.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.491651E-01 | lm-global-supervised-tokens: 1.258061E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.064877E-02 | lm-token-buffer-mib: 2.340102E+02 | global_load_balancing_loss: 9.986857E-01 | loss scale: 1.0 | grad norm: 0.175 | params norm: 11144.069 | params rms: 0.060342 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:14:44.650183 | finish at 2026-10-17 19:33:14 + [2026-10-17 12:22:15] iteration 2268/ 2384 | consumed samples: 72576 | elapsed time per iteration (ms): 225460.5 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.50% | FLOPs estimator: workload-aware | logical tokens/s: 74413.0 | valid tokens/s: 74413.0 | physical tokens/s: 75485.3 | loss tokens/s: 53143.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.517222E-01 | lm-global-supervised-tokens: 1.198177E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.417457E-02 | lm-token-buffer-mib: 2.374818E+02 | global_load_balancing_loss: 1.000476E+00 | loss scale: 1.0 | grad norm: 0.146 | params norm: 11144.042 | params rms: 0.060341 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:15:53.415539 | finish at 2026-10-17 19:38:08 + [2026-10-17 12:25:54] iteration 2269/ 2384 | consumed samples: 72608 | elapsed time per iteration (ms): 218944.8 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.01% | FLOPs estimator: workload-aware | logical tokens/s: 76627.5 | valid tokens/s: 76627.5 | physical tokens/s: 76981.0 | loss tokens/s: 54123.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.495222E-01 | lm-global-supervised-tokens: 1.185008E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.183711E-02 | lm-token-buffer-mib: 2.331129E+02 | global_load_balancing_loss: 1.000528E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11144.017 | params rms: 0.060341 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:59:38.657033 | finish at 2026-10-17 19:25:33 + [2026-10-17 12:29:56] iteration 2270/ 2384 | consumed samples: 72640 | elapsed time per iteration (ms): 242287.6 | throughput per GPU (TFLOP/s/GPU): 17.9 | MFU 5.74% | FLOPs estimator: workload-aware | logical tokens/s: 69245.0 | valid tokens/s: 69245.0 | physical tokens/s: 69780.2 | loss tokens/s: 53857.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.909510E-01 | lm-global-supervised-tokens: 1.304906E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.272904E-02 | lm-token-buffer-mib: 2.328821E+02 | global_load_balancing_loss: 1.000129E+00 | loss scale: 1.0 | grad norm: 0.154 | params norm: 11143.991 | params rms: 0.060341 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:40:20.784502 | finish at 2026-10-17 20:10:17 + [2026-10-17 12:33:39] iteration 2271/ 2384 | consumed samples: 72672 | elapsed time per iteration (ms): 223020.5 | throughput per GPU (TFLOP/s/GPU): 19.8 | MFU 6.35% | FLOPs estimator: workload-aware | logical tokens/s: 75227.2 | valid tokens/s: 75227.2 | physical tokens/s: 76260.9 | loss tokens/s: 58613.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.119193E-01 | lm-global-supervised-tokens: 1.307196E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.022280E-02 | lm-token-buffer-mib: 2.357020E+02 | global_load_balancing_loss: 1.001539E+00 | loss scale: 1.0 | grad norm: 0.511 | params norm: 11143.965 | params rms: 0.060341 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:00:01.311267 | finish at 2026-10-17 19:33:41 + [2026-10-17 12:37:20] iteration 2272/ 2384 | consumed samples: 72704 | elapsed time per iteration (ms): 221026.7 | throughput per GPU (TFLOP/s/GPU): 25.0 | MFU 8.01% | FLOPs estimator: workload-aware | logical tokens/s: 75905.6 | valid tokens/s: 75905.6 | physical tokens/s: 76340.5 | loss tokens/s: 58592.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.139641E-01 | lm-global-supervised-tokens: 1.295039E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.242259E-02 | lm-token-buffer-mib: 2.336378E+02 | global_load_balancing_loss: 1.000760E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11143.940 | params rms: 0.060341 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:52:34.993713 | finish at 2026-10-17 19:29:55 + [2026-10-17 12:41:02] iteration 2273/ 2384 | consumed samples: 72736 | elapsed time per iteration (ms): 221456.4 | throughput per GPU (TFLOP/s/GPU): 20.4 | MFU 6.53% | FLOPs estimator: workload-aware | logical tokens/s: 75758.5 | valid tokens/s: 75758.5 | physical tokens/s: 76748.4 | loss tokens/s: 58271.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.261549E-01 | lm-global-supervised-tokens: 1.290458E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.805943E-02 | lm-token-buffer-mib: 2.361598E+02 | global_load_balancing_loss: 9.996058E-01 | loss scale: 1.0 | grad norm: 0.452 | params norm: 11143.916 | params rms: 0.060341 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:49:41.659147 | finish at 2026-10-17 19:30:43 + [2026-10-17 12:44:42] iteration 2274/ 2384 | consumed samples: 72768 | elapsed time per iteration (ms): 219834.1 | throughput per GPU (TFLOP/s/GPU): 24.1 | MFU 7.71% | FLOPs estimator: workload-aware | logical tokens/s: 76317.5 | valid tokens/s: 76317.5 | physical tokens/s: 76732.8 | loss tokens/s: 57895.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.490885E-01 | lm-global-supervised-tokens: 1.272745E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.846433E-02 | lm-token-buffer-mib: 2.325734E+02 | global_load_balancing_loss: 1.000163E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11143.888 | params rms: 0.060341 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:43:01.746778 | finish at 2026-10-17 19:27:43 + [2026-10-17 12:48:23] iteration 2275/ 2384 | consumed samples: 72800 | elapsed time per iteration (ms): 221244.5 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.61% | FLOPs estimator: workload-aware | logical tokens/s: 75831.1 | valid tokens/s: 75831.1 | physical tokens/s: 76639.9 | loss tokens/s: 57852.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.897244E-01 | lm-global-supervised-tokens: 1.279951E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.976185E-02 | lm-token-buffer-mib: 2.351001E+02 | global_load_balancing_loss: 9.996148E-01 | loss scale: 1.0 | grad norm: 0.105 | params norm: 11143.859 | params rms: 0.060340 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:41:55.655039 | finish at 2026-10-17 19:30:18 + [2026-10-17 12:52:03] iteration 2276/ 2384 | consumed samples: 72832 | elapsed time per iteration (ms): 220179.0 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.84% | FLOPs estimator: workload-aware | logical tokens/s: 76198.0 | valid tokens/s: 76198.0 | physical tokens/s: 76616.1 | loss tokens/s: 56819.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.169997E-01 | lm-global-supervised-tokens: 1.251040E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.711416E-02 | lm-token-buffer-mib: 2.323102E+02 | global_load_balancing_loss: 1.000439E+00 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11143.827 | params rms: 0.060340 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:36:19.333457 | finish at 2026-10-17 19:28:22 + [2026-10-17 12:55:44] iteration 2277/ 2384 | consumed samples: 72864 | elapsed time per iteration (ms): 221199.8 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.51% | FLOPs estimator: workload-aware | logical tokens/s: 75846.3 | valid tokens/s: 75846.3 | physical tokens/s: 76499.4 | loss tokens/s: 57696.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.587345E-01 | lm-global-supervised-tokens: 1.276234E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.526522E-02 | lm-token-buffer-mib: 2.341816E+02 | global_load_balancing_loss: 1.000838E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11143.795 | params rms: 0.060340 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:34:28.381892 | finish at 2026-10-17 19:30:13 + [2026-10-17 12:59:25] iteration 2278/ 2384 | consumed samples: 72896 | elapsed time per iteration (ms): 221204.9 | throughput per GPU (TFLOP/s/GPU): 25.1 | MFU 8.04% | FLOPs estimator: workload-aware | logical tokens/s: 75844.4 | valid tokens/s: 75844.4 | physical tokens/s: 76326.7 | loss tokens/s: 56840.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.746645E-01 | lm-global-supervised-tokens: 1.257343E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.258635E-02 | lm-token-buffer-mib: 2.335439E+02 | global_load_balancing_loss: 1.000883E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11143.766 | params rms: 0.060340 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:30:47.719984 | finish at 2026-10-17 19:30:13 + [2026-10-17 13:03:11] iteration 2279/ 2384 | consumed samples: 72928 | elapsed time per iteration (ms): 226049.2 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.62% | FLOPs estimator: workload-aware | logical tokens/s: 74219.2 | valid tokens/s: 74219.2 | physical tokens/s: 75169.2 | loss tokens/s: 53947.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.584217E-01 | lm-global-supervised-tokens: 1.219468E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.712281E-02 | lm-token-buffer-mib: 2.343587E+02 | global_load_balancing_loss: 1.000250E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11143.733 | params rms: 0.060340 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:35:35.169486 | finish at 2026-10-17 19:38:47 + [2026-10-17 13:07:17] iteration 2280/ 2384 | consumed samples: 72960 | elapsed time per iteration (ms): 245450.6 | throughput per GPU (TFLOP/s/GPU): 16.8 | MFU 5.39% | FLOPs estimator: workload-aware | logical tokens/s: 68352.7 | valid tokens/s: 68352.7 | physical tokens/s: 69710.4 | loss tokens/s: 53051.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.295183E-01 | lm-global-supervised-tokens: 1.302160E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.598435E-02 | lm-token-buffer-mib: 2.378257E+02 | global_load_balancing_loss: 1.000650E+00 | loss scale: 1.0 | grad norm: 0.305 | params norm: 11143.701 | params rms: 0.060340 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:05:26.864151 | finish at 2026-10-17 20:12:44 +times across ranks (ms): + save-checkpoint: + rank 0: 24455.07 + rank 1: 24455.09 + rank 2: 24455.10 + rank 3: 24455.10 + rank 4: 24455.07 + rank 5: 24455.08 + rank 6: 24455.08 + rank 7: 24455.07 + rank 8: 24455.91 + rank 9: 24455.98 + rank 10: 24455.93 + rank 11: 24455.91 + rank 12: 24455.92 + rank 13: 24455.92 + rank 14: 24455.93 + rank 15: 24455.90 + rank 16: 24455.84 + rank 17: 24455.83 + rank 18: 24455.84 + rank 19: 24455.81 + rank 20: 24455.81 + rank 21: 24455.85 + rank 22: 24455.84 + rank 23: 24455.80 + rank 24: 24456.12 + rank 25: 24456.08 + rank 26: 24456.27 + rank 27: 24456.09 + rank 28: 24456.15 + rank 29: 24456.09 + rank 30: 24456.07 + rank 31: 24456.08 + rank 32: 24455.92 + rank 33: 24455.92 + rank 34: 24455.89 + rank 35: 24457.65 + rank 36: 24455.88 + rank 37: 24455.92 + rank 38: 24455.89 + rank 39: 24455.88 + rank 40: 24455.99 + rank 41: 24455.98 + rank 42: 24455.96 + rank 43: 24455.98 + rank 44: 24455.98 + rank 45: 24455.99 + rank 46: 24455.97 + rank 47: 24455.99 + rank 48: 24455.98 + rank 49: 24455.88 + rank 50: 24455.76 + rank 51: 24455.77 + rank 52: 24455.71 + rank 53: 24455.80 + rank 54: 24455.63 + rank 55: 24455.98 + rank 56: 24456.23 + rank 57: 24456.24 + rank 58: 24456.23 + rank 59: 24456.23 + rank 60: 24456.25 + rank 61: 24456.23 + rank 62: 24456.22 + rank 63: 24456.25 + [2026-10-17 13:11:21] iteration 2281/ 2384 | consumed samples: 72992 | elapsed time per iteration (ms): 219158.1 | throughput per GPU (TFLOP/s/GPU): 24.8 | MFU 7.95% | FLOPs estimator: workload-aware | logical tokens/s: 76552.9 | valid tokens/s: 76552.9 | physical tokens/s: 77698.6 | loss tokens/s: 59736.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.364090E-01 | lm-global-supervised-tokens: 1.309167E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 1.113067E-01 | lm-token-buffer-mib: 2.334048E+02 | global_load_balancing_loss: 1.000092E+00 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11143.672 | params rms: 0.060339 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:16:13.286572 | finish at 2026-10-17 19:27:34 + [2026-10-17 13:15:09] iteration 2282/ 2384 | consumed samples: 73024 | elapsed time per iteration (ms): 228599.7 | throughput per GPU (TFLOP/s/GPU): 20.9 | MFU 6.70% | FLOPs estimator: workload-aware | logical tokens/s: 73391.1 | valid tokens/s: 73391.1 | physical tokens/s: 73896.6 | loss tokens/s: 52498.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.528279E-01 | lm-global-supervised-tokens: 1.200105E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.698593E-02 | lm-token-buffer-mib: 2.331409E+02 | global_load_balancing_loss: 1.000112E+00 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11143.646 | params rms: 0.060339 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:28:37.166503 | finish at 2026-10-17 19:43:46 + [2026-10-17 13:18:51] iteration 2283/ 2384 | consumed samples: 73056 | elapsed time per iteration (ms): 221727.8 | throughput per GPU (TFLOP/s/GPU): 24.9 | MFU 7.99% | FLOPs estimator: workload-aware | logical tokens/s: 75665.7 | valid tokens/s: 75665.7 | physical tokens/s: 76177.0 | loss tokens/s: 62400.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.289020E-01 | lm-global-supervised-tokens: 1.383593E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.220312E-02 | lm-token-buffer-mib: 2.337020E+02 | global_load_balancing_loss: 9.992660E-01 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11143.617 | params rms: 0.060339 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:13:14.508728 | finish at 2026-10-17 19:32:05 + [2026-10-17 13:22:32] iteration 2284/ 2384 | consumed samples: 73088 | elapsed time per iteration (ms): 220705.5 | throughput per GPU (TFLOP/s/GPU): 25.7 | MFU 8.25% | FLOPs estimator: workload-aware | logical tokens/s: 76016.2 | valid tokens/s: 76016.2 | physical tokens/s: 76413.9 | loss tokens/s: 58153.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.891984E-01 | lm-global-supervised-tokens: 1.283485E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.155112E-02 | lm-token-buffer-mib: 2.332994E+02 | global_load_balancing_loss: 1.001357E+00 | loss scale: 1.0 | grad norm: 0.130 | params norm: 11143.586 | params rms: 0.060339 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:07:50.547628 | finish at 2026-10-17 19:30:22 + [2026-10-17 13:26:15] iteration 2285/ 2384 | consumed samples: 73120 | elapsed time per iteration (ms): 223005.0 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.85% | FLOPs estimator: workload-aware | logical tokens/s: 75232.4 | valid tokens/s: 75232.4 | physical tokens/s: 75999.0 | loss tokens/s: 54609.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.354436E-01 | lm-global-supervised-tokens: 1.217812E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.253399E-02 | lm-token-buffer-mib: 2.325742E+02 | global_load_balancing_loss: 1.000973E+00 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11143.553 | params rms: 0.060339 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:07:57.493005 | finish at 2026-10-17 19:34:12 + [2026-10-17 13:30:00] iteration 2286/ 2384 | consumed samples: 73152 | elapsed time per iteration (ms): 225171.0 | throughput per GPU (TFLOP/s/GPU): 19.9 | MFU 6.38% | FLOPs estimator: workload-aware | logical tokens/s: 74508.7 | valid tokens/s: 74508.7 | physical tokens/s: 75549.3 | loss tokens/s: 57031.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.113281E-01 | lm-global-supervised-tokens: 1.284180E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.260405E-02 | lm-token-buffer-mib: 2.359061E+02 | global_load_balancing_loss: 1.000389E+00 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11143.523 | params rms: 0.060339 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:07:46.762253 | finish at 2026-10-17 19:37:47 + [2026-10-17 13:33:42] iteration 2287/ 2384 | consumed samples: 73184 | elapsed time per iteration (ms): 222437.4 | throughput per GPU (TFLOP/s/GPU): 24.8 | MFU 7.94% | FLOPs estimator: workload-aware | logical tokens/s: 75424.3 | valid tokens/s: 75424.3 | physical tokens/s: 75859.8 | loss tokens/s: 58429.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.149831E-01 | lm-global-supervised-tokens: 1.299691E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.555120E-02 | lm-token-buffer-mib: 2.323913E+02 | global_load_balancing_loss: 9.992817E-01 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11143.493 | params rms: 0.060338 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:59:36.423347 | finish at 2026-10-17 19:33:19 + [2026-10-17 13:37:27] iteration 2288/ 2384 | consumed samples: 73216 | elapsed time per iteration (ms): 224708.9 | throughput per GPU (TFLOP/s/GPU): 19.9 | MFU 6.38% | FLOPs estimator: workload-aware | logical tokens/s: 74661.9 | valid tokens/s: 74661.9 | physical tokens/s: 75593.1 | loss tokens/s: 57803.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.612789E-01 | lm-global-supervised-tokens: 1.298902E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.528054E-02 | lm-token-buffer-mib: 2.331384E+02 | global_load_balancing_loss: 1.000718E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11143.462 | params rms: 0.060338 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:59:32.052956 | finish at 2026-10-17 19:36:59 + [2026-10-17 13:41:07] iteration 2289/ 2384 | consumed samples: 73248 | elapsed time per iteration (ms): 220394.0 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.92% | FLOPs estimator: workload-aware | logical tokens/s: 76123.6 | valid tokens/s: 76123.6 | physical tokens/s: 76526.2 | loss tokens/s: 53840.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.439551E-01 | lm-global-supervised-tokens: 1.186600E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.149053E-02 | lm-token-buffer-mib: 2.328395E+02 | global_load_balancing_loss: 9.997050E-01 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11143.429 | params rms: 0.060338 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:48:57.432179 | finish at 2026-10-17 19:30:05 + [2026-10-17 13:45:12] iteration 2290/ 2384 | consumed samples: 73280 | elapsed time per iteration (ms): 245183.0 | throughput per GPU (TFLOP/s/GPU): 20.2 | MFU 6.48% | FLOPs estimator: workload-aware | logical tokens/s: 68427.1 | valid tokens/s: 68427.1 | physical tokens/s: 69137.1 | loss tokens/s: 50334.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.253222E-01 | lm-global-supervised-tokens: 1.234112E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.429187E-02 | lm-token-buffer-mib: 2.336276E+02 | global_load_balancing_loss: 1.000331E+00 | loss scale: 1.0 | grad norm: 0.164 | params norm: 11143.395 | params rms: 0.060338 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:24:07.198669 | finish at 2026-10-17 20:09:20 + [2026-10-17 13:48:54] iteration 2291/ 2384 | consumed samples: 73312 | elapsed time per iteration (ms): 221562.5 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.34% | FLOPs estimator: workload-aware | logical tokens/s: 75722.1 | valid tokens/s: 75722.1 | physical tokens/s: 76532.8 | loss tokens/s: 57487.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.228854E-01 | lm-global-supervised-tokens: 1.273715E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.664554E-02 | lm-token-buffer-mib: 2.355691E+02 | global_load_balancing_loss: 9.999793E-01 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11143.361 | params rms: 0.060338 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:43:25.315693 | finish at 2026-10-17 19:32:19 + [2026-10-17 13:52:34] iteration 2292/ 2384 | consumed samples: 73344 | elapsed time per iteration (ms): 220343.9 | throughput per GPU (TFLOP/s/GPU): 25.7 | MFU 8.24% | FLOPs estimator: workload-aware | logical tokens/s: 76141.0 | valid tokens/s: 76141.0 | physical tokens/s: 76511.7 | loss tokens/s: 53016.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.174196E-01 | lm-global-supervised-tokens: 1.168191E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.144316E-02 | lm-token-buffer-mib: 2.332882E+02 | global_load_balancing_loss: 1.000248E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11143.332 | params rms: 0.060338 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:37:51.640793 | finish at 2026-10-17 19:30:26 + [2026-10-17 13:56:19] iteration 2293/ 2384 | consumed samples: 73376 | elapsed time per iteration (ms): 224842.4 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.83% | FLOPs estimator: workload-aware | logical tokens/s: 74617.6 | valid tokens/s: 74617.6 | physical tokens/s: 75367.4 | loss tokens/s: 53823.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.829046E-01 | lm-global-supervised-tokens: 1.210177E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.598059E-02 | lm-token-buffer-mib: 2.341285E+02 | global_load_balancing_loss: 1.002222E+00 | loss scale: 1.0 | grad norm: 0.129 | params norm: 11143.303 | params rms: 0.060337 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:41:00.659123 | finish at 2026-10-17 19:37:20 + [2026-10-17 14:00:05] iteration 2294/ 2384 | consumed samples: 73408 | elapsed time per iteration (ms): 226071.8 | throughput per GPU (TFLOP/s/GPU): 25.1 | MFU 8.05% | FLOPs estimator: workload-aware | logical tokens/s: 74211.7 | valid tokens/s: 74211.7 | physical tokens/s: 74633.4 | loss tokens/s: 56312.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.947216E-01 | lm-global-supervised-tokens: 1.273064E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.237818E-02 | lm-token-buffer-mib: 2.331212E+02 | global_load_balancing_loss: 1.001182E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11143.275 | params rms: 0.060337 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:39:06.458824 | finish at 2026-10-17 19:39:12 + [2026-10-17 14:03:48] iteration 2295/ 2384 | consumed samples: 73440 | elapsed time per iteration (ms): 222781.6 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.38% | FLOPs estimator: workload-aware | logical tokens/s: 75307.9 | valid tokens/s: 75307.9 | physical tokens/s: 75705.5 | loss tokens/s: 60359.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.577833E-01 | lm-global-supervised-tokens: 1.344699E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.129873E-02 | lm-token-buffer-mib: 2.327178E+02 | global_load_balancing_loss: 1.000868E+00 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11143.247 | params rms: 0.060337 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:30:27.564607 | finish at 2026-10-17 19:34:16 + [2026-10-17 14:07:28] iteration 2296/ 2384 | consumed samples: 73472 | elapsed time per iteration (ms): 220135.2 | throughput per GPU (TFLOP/s/GPU): 21.1 | MFU 6.75% | FLOPs estimator: workload-aware | logical tokens/s: 76213.1 | valid tokens/s: 76213.1 | physical tokens/s: 76554.6 | loss tokens/s: 53952.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.871600E-01 | lm-global-supervised-tokens: 1.187684E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.555639E-02 | lm-token-buffer-mib: 2.327280E+02 | global_load_balancing_loss: 9.993801E-01 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11143.221 | params rms: 0.060337 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:22:51.896553 | finish at 2026-10-17 19:30:20 + [2026-10-17 14:11:10] iteration 2297/ 2384 | consumed samples: 73504 | elapsed time per iteration (ms): 221756.2 | throughput per GPU (TFLOP/s/GPU): 25.1 | MFU 8.06% | FLOPs estimator: workload-aware | logical tokens/s: 75656.0 | valid tokens/s: 75656.0 | physical tokens/s: 76608.5 | loss tokens/s: 53646.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.223454E-01 | lm-global-supervised-tokens: 1.189643E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.254446E-02 | lm-token-buffer-mib: 2.342397E+02 | global_load_balancing_loss: 9.994433E-01 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11143.199 | params rms: 0.060337 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:21:32.792808 | finish at 2026-10-17 19:32:43 + [2026-10-17 14:14:56] iteration 2298/ 2384 | consumed samples: 73536 | elapsed time per iteration (ms): 226355.1 | throughput per GPU (TFLOP/s/GPU): 24.5 | MFU 7.85% | FLOPs estimator: workload-aware | logical tokens/s: 74118.9 | valid tokens/s: 74118.9 | physical tokens/s: 75084.6 | loss tokens/s: 51888.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.180083E-01 | lm-global-supervised-tokens: 1.174529E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.522236E-02 | lm-token-buffer-mib: 2.368417E+02 | global_load_balancing_loss: 9.990797E-01 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11143.175 | params rms: 0.060337 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:24:26.534718 | finish at 2026-10-17 19:39:23 + [2026-10-17 14:18:40] iteration 2299/ 2384 | consumed samples: 73568 | elapsed time per iteration (ms): 223327.0 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.00% | FLOPs estimator: workload-aware | logical tokens/s: 75123.8 | valid tokens/s: 75123.8 | physical tokens/s: 75889.4 | loss tokens/s: 56563.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.064775E-01 | lm-global-supervised-tokens: 1.263207E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.863480E-02 | lm-token-buffer-mib: 2.332564E+02 | global_load_balancing_loss: 9.996828E-01 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11143.148 | params rms: 0.060337 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:16:22.795215 | finish at 2026-10-17 19:35:02 + [2026-10-17 14:22:47] iteration 2300/ 2384 | consumed samples: 73600 | elapsed time per iteration (ms): 247540.3 | throughput per GPU (TFLOP/s/GPU): 19.2 | MFU 6.15% | FLOPs estimator: workload-aware | logical tokens/s: 67775.7 | valid tokens/s: 67775.7 | physical tokens/s: 69468.1 | loss tokens/s: 51142.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.045185E-01 | lm-global-supervised-tokens: 1.265992E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.308685E-02 | lm-token-buffer-mib: 2.375474E+02 | global_load_balancing_loss: 1.001384E+00 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11143.120 | params rms: 0.060336 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:46:33.383529 | finish at 2026-10-17 20:09:21 + [2026-10-17 14:26:32] iteration 2301/ 2384 | consumed samples: 73632 | elapsed time per iteration (ms): 224360.9 | throughput per GPU (TFLOP/s/GPU): 24.3 | MFU 7.78% | FLOPs estimator: workload-aware | logical tokens/s: 74777.7 | valid tokens/s: 74777.7 | physical tokens/s: 75756.8 | loss tokens/s: 54823.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.038017E-01 | lm-global-supervised-tokens: 1.230014E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.497056E-02 | lm-token-buffer-mib: 2.353905E+02 | global_load_balancing_loss: 1.000455E+00 | loss scale: 1.0 | grad norm: 0.169 | params norm: 11143.093 | params rms: 0.060336 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:10:21.957979 | finish at 2026-10-17 19:36:54 + [2026-10-17 14:30:12] iteration 2302/ 2384 | consumed samples: 73664 | elapsed time per iteration (ms): 220219.0 | throughput per GPU (TFLOP/s/GPU): 23.5 | MFU 7.53% | FLOPs estimator: workload-aware | logical tokens/s: 76184.1 | valid tokens/s: 76184.1 | physical tokens/s: 76604.8 | loss tokens/s: 59625.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.226683E-01 | lm-global-supervised-tokens: 1.313061E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.561822E-02 | lm-token-buffer-mib: 2.326640E+02 | global_load_balancing_loss: 1.002509E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11143.067 | params rms: 0.060336 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:00:57.960100 | finish at 2026-10-17 19:31:10 + [2026-10-17 14:33:56] iteration 2303/ 2384 | consumed samples: 73696 | elapsed time per iteration (ms): 224515.3 | throughput per GPU (TFLOP/s/GPU): 24.2 | MFU 7.76% | FLOPs estimator: workload-aware | logical tokens/s: 74726.3 | valid tokens/s: 74726.3 | physical tokens/s: 75674.8 | loss tokens/s: 56920.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.060522E-01 | lm-global-supervised-tokens: 1.277960E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.134936E-02 | lm-token-buffer-mib: 2.346332E+02 | global_load_balancing_loss: 1.000821E+00 | loss scale: 1.0 | grad norm: 0.269 | params norm: 11143.039 | params rms: 0.060336 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:03:05.742895 | finish at 2026-10-17 19:37:02 + [2026-10-17 14:37:38] iteration 2304/ 2384 | consumed samples: 73728 | elapsed time per iteration (ms): 221528.1 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.38% | FLOPs estimator: workload-aware | logical tokens/s: 75733.9 | valid tokens/s: 75733.9 | physical tokens/s: 76449.3 | loss tokens/s: 58403.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.139326E-01 | lm-global-supervised-tokens: 1.293804E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.527186E-02 | lm-token-buffer-mib: 2.345786E+02 | global_load_balancing_loss: 1.001765E+00 | loss scale: 1.0 | grad norm: 0.114 | params norm: 11143.011 | params rms: 0.060336 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:55:22.251587 | finish at 2026-10-17 19:33:00 + [2026-10-17 14:41:19] iteration 2305/ 2384 | consumed samples: 73760 | elapsed time per iteration (ms): 220915.4 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.20% | FLOPs estimator: workload-aware | logical tokens/s: 75943.9 | valid tokens/s: 75943.9 | physical tokens/s: 76692.7 | loss tokens/s: 57551.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.792441E-01 | lm-global-supervised-tokens: 1.271408E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.459019E-02 | lm-token-buffer-mib: 2.340327E+02 | global_load_balancing_loss: 1.000462E+00 | loss scale: 1.0 | grad norm: 0.235 | params norm: 11142.984 | params rms: 0.060336 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:50:52.320068 | finish at 2026-10-17 19:32:11 + [2026-10-17 14:45:06] iteration 2306/ 2384 | consumed samples: 73792 | elapsed time per iteration (ms): 227255.5 | throughput per GPU (TFLOP/s/GPU): 19.7 | MFU 6.32% | FLOPs estimator: workload-aware | logical tokens/s: 73825.3 | valid tokens/s: 73825.3 | physical tokens/s: 74831.6 | loss tokens/s: 54876.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.866845E-01 | lm-global-supervised-tokens: 1.247103E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.743218E-02 | lm-token-buffer-mib: 2.362440E+02 | global_load_balancing_loss: 1.000930E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11142.957 | params rms: 0.060335 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:55:25.932279 | finish at 2026-10-17 19:40:32 + [2026-10-17 14:48:52] iteration 2307/ 2384 | consumed samples: 73824 | elapsed time per iteration (ms): 226389.5 | throughput per GPU (TFLOP/s/GPU): 21.8 | MFU 6.97% | FLOPs estimator: workload-aware | logical tokens/s: 74107.6 | valid tokens/s: 74107.6 | physical tokens/s: 74608.7 | loss tokens/s: 56267.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.210357E-01 | lm-global-supervised-tokens: 1.273847E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.131558E-02 | lm-token-buffer-mib: 2.337512E+02 | global_load_balancing_loss: 1.000165E+00 | loss scale: 1.0 | grad norm: 0.259 | params norm: 11142.931 | params rms: 0.060335 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:50:31.992300 | finish at 2026-10-17 19:39:24 + [2026-10-17 14:52:41] iteration 2308/ 2384 | consumed samples: 73856 | elapsed time per iteration (ms): 229083.2 | throughput per GPU (TFLOP/s/GPU): 25.1 | MFU 8.05% | FLOPs estimator: workload-aware | logical tokens/s: 73236.1 | valid tokens/s: 73236.1 | physical tokens/s: 74648.3 | loss tokens/s: 55836.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.880574E-01 | lm-global-supervised-tokens: 1.279118E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.593105E-02 | lm-token-buffer-mib: 2.370252E+02 | global_load_balancing_loss: 9.988361E-01 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11142.905 | params rms: 0.060335 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:50:10.323778 | finish at 2026-10-17 19:42:52 + [2026-10-17 14:56:24] iteration 2309/ 2384 | consumed samples: 73888 | elapsed time per iteration (ms): 222803.7 | throughput per GPU (TFLOP/s/GPU): 21.0 | MFU 6.72% | FLOPs estimator: workload-aware | logical tokens/s: 75300.3 | valid tokens/s: 75300.3 | physical tokens/s: 76066.5 | loss tokens/s: 53423.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.343751E-01 | lm-global-supervised-tokens: 1.190292E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.933865E-02 | lm-token-buffer-mib: 2.323668E+02 | global_load_balancing_loss: 9.994454E-01 | loss scale: 1.0 | grad norm: 0.238 | params norm: 11142.877 | params rms: 0.060335 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:38:30.274386 | finish at 2026-10-17 19:34:55 + [2026-10-17 15:00:28] iteration 2310/ 2384 | consumed samples: 73920 | elapsed time per iteration (ms): 243272.4 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.97% | FLOPs estimator: workload-aware | logical tokens/s: 68964.6 | valid tokens/s: 68964.6 | physical tokens/s: 69807.4 | loss tokens/s: 50433.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.468100E-01 | lm-global-supervised-tokens: 1.226904E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.376603E-02 | lm-token-buffer-mib: 2.339773E+02 | global_load_balancing_loss: 1.000479E+00 | loss scale: 1.0 | grad norm: 0.137 | params norm: 11142.846 | params rms: 0.060335 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:00:02.155264 | finish at 2026-10-17 20:00:30 + [2026-10-17 15:04:07] iteration 2311/ 2384 | consumed samples: 73952 | elapsed time per iteration (ms): 219822.0 | throughput per GPU (TFLOP/s/GPU): 25.4 | MFU 8.14% | FLOPs estimator: workload-aware | logical tokens/s: 76321.7 | valid tokens/s: 76321.7 | physical tokens/s: 77097.1 | loss tokens/s: 56946.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.398970E-01 | lm-global-supervised-tokens: 1.251814E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.563748E-02 | lm-token-buffer-mib: 2.333123E+02 | global_load_balancing_loss: 1.000817E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11142.817 | params rms: 0.060335 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:27:27.004505 | finish at 2026-10-17 19:31:34 + [2026-10-17 15:07:51] iteration 2312/ 2384 | consumed samples: 73984 | elapsed time per iteration (ms): 223593.4 | throughput per GPU (TFLOP/s/GPU): 23.7 | MFU 7.59% | FLOPs estimator: workload-aware | logical tokens/s: 75034.3 | valid tokens/s: 75034.3 | physical tokens/s: 75507.9 | loss tokens/s: 56914.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.744389E-01 | lm-global-supervised-tokens: 1.272562E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.293229E-02 | lm-token-buffer-mib: 2.338450E+02 | global_load_balancing_loss: 9.997063E-01 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11142.791 | params rms: 0.060335 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:28:18.727581 | finish at 2026-10-17 19:36:10 + [2026-10-17 15:11:32] iteration 2313/ 2384 | consumed samples: 74016 | elapsed time per iteration (ms): 220914.3 | throughput per GPU (TFLOP/s/GPU): 20.1 | MFU 6.45% | FLOPs estimator: workload-aware | logical tokens/s: 75944.4 | valid tokens/s: 75944.4 | physical tokens/s: 76462.2 | loss tokens/s: 55675.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.420967E-01 | lm-global-supervised-tokens: 1.229947E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.536924E-02 | lm-token-buffer-mib: 2.339887E+02 | global_load_balancing_loss: 1.000504E+00 | loss scale: 1.0 | grad norm: 0.183 | params norm: 11142.765 | params rms: 0.060334 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:21:24.911793 | finish at 2026-10-17 19:32:57 + [2026-10-17 15:15:12] iteration 2314/ 2384 | consumed samples: 74048 | elapsed time per iteration (ms): 220152.9 | throughput per GPU (TFLOP/s/GPU): 17.6 | MFU 5.63% | FLOPs estimator: workload-aware | logical tokens/s: 76207.1 | valid tokens/s: 76207.1 | physical tokens/s: 77120.5 | loss tokens/s: 56142.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.945115E-01 | lm-global-supervised-tokens: 1.236001E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.561167E-02 | lm-token-buffer-mib: 2.343414E+02 | global_load_balancing_loss: 1.000389E+00 | loss scale: 1.0 | grad norm: 0.164 | params norm: 11142.740 | params rms: 0.060334 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:16:50.705051 | finish at 2026-10-17 19:32:03 + [2026-10-17 15:18:54] iteration 2315/ 2384 | consumed samples: 74080 | elapsed time per iteration (ms): 221695.2 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.22% | FLOPs estimator: workload-aware | logical tokens/s: 75676.9 | valid tokens/s: 75676.9 | physical tokens/s: 76038.1 | loss tokens/s: 56184.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.243334E-01 | lm-global-supervised-tokens: 1.245584E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.272478E-02 | lm-token-buffer-mib: 2.328731E+02 | global_load_balancing_loss: 1.001852E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11142.719 | params rms: 0.060334 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:14:56.968107 | finish at 2026-10-17 19:33:51 + [2026-10-17 15:22:34] iteration 2316/ 2384 | consumed samples: 74112 | elapsed time per iteration (ms): 220679.8 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.87% | FLOPs estimator: workload-aware | logical tokens/s: 76025.1 | valid tokens/s: 76025.1 | physical tokens/s: 76927.4 | loss tokens/s: 57459.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.761549E-01 | lm-global-supervised-tokens: 1.268020E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.354721E-02 | lm-token-buffer-mib: 2.334173E+02 | global_load_balancing_loss: 9.994774E-01 | loss scale: 1.0 | grad norm: 0.433 | params norm: 11142.699 | params rms: 0.060334 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:10:06.225689 | finish at 2026-10-17 19:32:41 + [2026-10-17 15:26:19] iteration 2317/ 2384 | consumed samples: 74144 | elapsed time per iteration (ms): 224222.7 | throughput per GPU (TFLOP/s/GPU): 25.7 | MFU 8.24% | FLOPs estimator: workload-aware | logical tokens/s: 74823.7 | valid tokens/s: 74823.7 | physical tokens/s: 75296.3 | loss tokens/s: 57827.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.397504E-01 | lm-global-supervised-tokens: 1.296621E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.010002E-02 | lm-token-buffer-mib: 2.335825E+02 | global_load_balancing_loss: 9.999289E-01 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11142.678 | params rms: 0.060334 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:10:22.923671 | finish at 2026-10-17 19:36:42 + [2026-10-17 15:29:59] iteration 2318/ 2384 | consumed samples: 74176 | elapsed time per iteration (ms): 220382.6 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.93% | FLOPs estimator: workload-aware | logical tokens/s: 76127.6 | valid tokens/s: 76127.6 | physical tokens/s: 76590.3 | loss tokens/s: 58933.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.694113E-01 | lm-global-supervised-tokens: 1.298783E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.136398E-02 | lm-token-buffer-mib: 2.335772E+02 | global_load_balancing_loss: 1.000900E+00 | loss scale: 1.0 | grad norm: 0.130 | params norm: 11142.656 | params rms: 0.060334 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:02:25.251148 | finish at 2026-10-17 19:32:24 + [2026-10-17 15:33:41] iteration 2319/ 2384 | consumed samples: 74208 | elapsed time per iteration (ms): 221634.1 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.12% | FLOPs estimator: workload-aware | logical tokens/s: 75697.8 | valid tokens/s: 75697.8 | physical tokens/s: 76292.4 | loss tokens/s: 56863.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.824160E-01 | lm-global-supervised-tokens: 1.260288E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.532175E-02 | lm-token-buffer-mib: 2.335546E+02 | global_load_balancing_loss: 1.000881E+00 | loss scale: 1.0 | grad norm: 0.152 | params norm: 11142.630 | params rms: 0.060334 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:00:06.216916 | finish at 2026-10-17 19:33:47 + [2026-10-17 15:37:49] iteration 2320/ 2384 | consumed samples: 74240 | elapsed time per iteration (ms): 248417.0 | throughput per GPU (TFLOP/s/GPU): 20.8 | MFU 6.67% | FLOPs estimator: workload-aware | logical tokens/s: 67536.4 | valid tokens/s: 67536.4 | physical tokens/s: 67969.3 | loss tokens/s: 51848.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.987627E-01 | lm-global-supervised-tokens: 1.288000E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.198783E-02 | lm-token-buffer-mib: 2.329819E+02 | global_load_balancing_loss: 9.997357E-01 | loss scale: 1.0 | grad norm: 0.153 | params norm: 11142.600 | params rms: 0.060334 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:24:58.690857 | finish at 2026-10-17 20:02:48 +times across ranks (ms): + save-checkpoint: + rank 0: 36303.55 + rank 1: 36303.59 + rank 2: 36303.59 + rank 3: 36303.58 + rank 4: 36303.55 + rank 5: 36303.57 + rank 6: 36303.58 + rank 7: 36303.56 + rank 8: 36302.62 + rank 9: 36302.69 + rank 10: 36302.79 + rank 11: 36302.66 + rank 12: 36302.72 + rank 13: 36302.59 + rank 14: 36302.74 + rank 15: 36302.66 + rank 16: 36303.46 + rank 17: 36303.50 + rank 18: 36303.46 + rank 19: 36303.46 + rank 20: 36303.44 + rank 21: 36303.47 + rank 22: 36303.46 + rank 23: 36303.43 + rank 24: 36303.64 + rank 25: 36303.61 + rank 26: 36303.63 + rank 27: 36303.60 + rank 28: 36303.59 + rank 29: 36304.47 + rank 30: 36303.61 + rank 31: 36303.61 + rank 32: 36303.33 + rank 33: 36303.25 + rank 34: 36303.29 + rank 35: 36303.31 + rank 36: 36303.22 + rank 37: 36303.28 + rank 38: 36303.24 + rank 39: 36303.24 + rank 40: 36301.75 + rank 41: 36301.75 + rank 42: 36301.74 + rank 43: 36301.79 + rank 44: 36301.70 + rank 45: 36301.71 + rank 46: 36301.58 + rank 47: 36301.72 + rank 48: 36303.36 + rank 49: 36303.29 + rank 50: 36303.39 + rank 51: 36303.38 + rank 52: 36303.33 + rank 53: 36303.44 + rank 54: 36303.32 + rank 55: 36303.30 + rank 56: 36303.53 + rank 57: 36303.58 + rank 58: 36303.40 + rank 59: 36303.57 + rank 60: 36303.58 + rank 61: 36303.54 + rank 62: 36303.55 + rank 63: 36303.53 + [2026-10-17 15:42:20] iteration 2321/ 2384 | consumed samples: 74272 | elapsed time per iteration (ms): 229958.6 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.14% | FLOPs estimator: workload-aware | logical tokens/s: 72957.4 | valid tokens/s: 72957.4 | physical tokens/s: 73293.5 | loss tokens/s: 53280.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.565833E-01 | lm-global-supervised-tokens: 1.225232E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.579025E-02 | lm-token-buffer-mib: 2.327137E+02 | global_load_balancing_loss: 1.000905E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11142.573 | params rms: 0.060333 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:01:27.389460 | finish at 2026-10-17 19:43:48 + [2026-10-17 15:46:10] iteration 2322/ 2384 | consumed samples: 74304 | elapsed time per iteration (ms): 229416.6 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.96% | FLOPs estimator: workload-aware | logical tokens/s: 73129.9 | valid tokens/s: 73129.9 | physical tokens/s: 74593.9 | loss tokens/s: 54696.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.530672E-01 | lm-global-supervised-tokens: 1.254838E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.995479E-02 | lm-token-buffer-mib: 2.353427E+02 | global_load_balancing_loss: 1.000762E+00 | loss scale: 1.0 | grad norm: 0.127 | params norm: 11142.551 | params rms: 0.060333 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:57:03.831180 | finish at 2026-10-17 19:43:14 + [2026-10-17 15:49:51] iteration 2323/ 2384 | consumed samples: 74336 | elapsed time per iteration (ms): 221011.8 | throughput per GPU (TFLOP/s/GPU): 21.7 | MFU 6.94% | FLOPs estimator: workload-aware | logical tokens/s: 75910.8 | valid tokens/s: 75910.8 | physical tokens/s: 76786.9 | loss tokens/s: 59353.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.614026E-01 | lm-global-supervised-tokens: 1.311792E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.104485E-02 | lm-token-buffer-mib: 2.341557E+02 | global_load_balancing_loss: 1.000097E+00 | loss scale: 1.0 | grad norm: 0.128 | params norm: 11142.527 | params rms: 0.060333 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:44:41.719498 | finish at 2026-10-17 19:34:32 + [2026-10-17 15:53:30] iteration 2324/ 2384 | consumed samples: 74368 | elapsed time per iteration (ms): 218765.4 | throughput per GPU (TFLOP/s/GPU): 19.0 | MFU 6.08% | FLOPs estimator: workload-aware | logical tokens/s: 76690.4 | valid tokens/s: 76690.4 | physical tokens/s: 77305.1 | loss tokens/s: 56245.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.386979E-01 | lm-global-supervised-tokens: 1.230466E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.188881E-02 | lm-token-buffer-mib: 2.334958E+02 | global_load_balancing_loss: 9.994435E-01 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11142.500 | params rms: 0.060333 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:38:45.921221 | finish at 2026-10-17 19:32:15 + [2026-10-17 15:57:12] iteration 2325/ 2384 | consumed samples: 74400 | elapsed time per iteration (ms): 222177.3 | throughput per GPU (TFLOP/s/GPU): 21.5 | MFU 6.88% | FLOPs estimator: workload-aware | logical tokens/s: 75512.6 | valid tokens/s: 75512.6 | physical tokens/s: 75913.7 | loss tokens/s: 56758.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.969793E-01 | lm-global-supervised-tokens: 1.261041E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.608286E-02 | lm-token-buffer-mib: 2.330623E+02 | global_load_balancing_loss: 1.000032E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11142.471 | params rms: 0.060333 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:38:28.459025 | finish at 2026-10-17 19:35:40 + [2026-10-17 16:00:51] iteration 2326/ 2384 | consumed samples: 74432 | elapsed time per iteration (ms): 219097.9 | throughput per GPU (TFLOP/s/GPU): 22.9 | MFU 7.33% | FLOPs estimator: workload-aware | logical tokens/s: 76574.0 | valid tokens/s: 76574.0 | physical tokens/s: 76898.0 | loss tokens/s: 55563.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.678597E-01 | lm-global-supervised-tokens: 1.217386E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.225894E-02 | lm-token-buffer-mib: 2.328466E+02 | global_load_balancing_loss: 1.000828E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11142.441 | params rms: 0.060333 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:31:47.678762 | finish at 2026-10-17 19:32:38 + [2026-10-17 16:04:30] iteration 2327/ 2384 | consumed samples: 74464 | elapsed time per iteration (ms): 219356.4 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.60% | FLOPs estimator: workload-aware | logical tokens/s: 76483.8 | valid tokens/s: 76483.8 | physical tokens/s: 77219.4 | loss tokens/s: 59733.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.659889E-01 | lm-global-supervised-tokens: 1.310292E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.370992E-02 | lm-token-buffer-mib: 2.345454E+02 | global_load_balancing_loss: 1.000194E+00 | loss scale: 1.0 | grad norm: 0.102 | params norm: 11142.410 | params rms: 0.060333 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:28:23.313442 | finish at 2026-10-17 19:32:53 + [2026-10-17 16:08:09] iteration 2328/ 2384 | consumed samples: 74496 | elapsed time per iteration (ms): 219319.7 | throughput per GPU (TFLOP/s/GPU): 18.8 | MFU 6.02% | FLOPs estimator: workload-aware | logical tokens/s: 76496.5 | valid tokens/s: 76496.5 | physical tokens/s: 77227.0 | loss tokens/s: 57607.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.356563E-01 | lm-global-supervised-tokens: 1.263438E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.509279E-02 | lm-token-buffer-mib: 2.336019E+02 | global_load_balancing_loss: 1.000604E+00 | loss scale: 1.0 | grad norm: 0.207 | params norm: 11142.379 | params rms: 0.060332 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:24:41.904362 | finish at 2026-10-17 19:32:51 + [2026-10-17 16:11:51] iteration 2329/ 2384 | consumed samples: 74528 | elapsed time per iteration (ms): 221477.6 | throughput per GPU (TFLOP/s/GPU): 19.4 | MFU 6.22% | FLOPs estimator: workload-aware | logical tokens/s: 75751.2 | valid tokens/s: 75751.2 | physical tokens/s: 76335.3 | loss tokens/s: 60117.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.960168E-01 | lm-global-supervised-tokens: 1.331460E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.609985E-02 | lm-token-buffer-mib: 2.337760E+02 | global_load_balancing_loss: 1.000257E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11142.350 | params rms: 0.060332 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:23:01.269540 | finish at 2026-10-17 19:34:52 + [2026-10-17 16:15:57] iteration 2330/ 2384 | consumed samples: 74560 | elapsed time per iteration (ms): 245585.8 | throughput per GPU (TFLOP/s/GPU): 20.7 | MFU 6.62% | FLOPs estimator: workload-aware | logical tokens/s: 68314.8 | valid tokens/s: 68314.8 | physical tokens/s: 68774.0 | loss tokens/s: 52462.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.856417E-01 | lm-global-supervised-tokens: 1.288399E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.156135E-02 | lm-token-buffer-mib: 2.341425E+02 | global_load_balancing_loss: 1.000507E+00 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11142.319 | params rms: 0.060332 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:41:01.632720 | finish at 2026-10-17 19:56:58 + [2026-10-17 16:19:36] iteration 2331/ 2384 | consumed samples: 74592 | elapsed time per iteration (ms): 219649.5 | throughput per GPU (TFLOP/s/GPU): 20.3 | MFU 6.50% | FLOPs estimator: workload-aware | logical tokens/s: 76381.6 | valid tokens/s: 76381.6 | physical tokens/s: 77217.4 | loss tokens/s: 59750.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.341751E-01 | lm-global-supervised-tokens: 1.312418E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.954121E-02 | lm-token-buffer-mib: 2.337741E+02 | global_load_balancing_loss: 1.000044E+00 | loss scale: 1.0 | grad norm: 0.175 | params norm: 11142.286 | params rms: 0.060332 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:14:01.421068 | finish at 2026-10-17 19:33:38 + [2026-10-17 16:23:21] iteration 2332/ 2384 | consumed samples: 74624 | elapsed time per iteration (ms): 224805.8 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.42% | FLOPs estimator: workload-aware | logical tokens/s: 74629.7 | valid tokens/s: 74629.7 | physical tokens/s: 75515.8 | loss tokens/s: 59408.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.581668E-01 | lm-global-supervised-tokens: 1.335530E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.531184E-02 | lm-token-buffer-mib: 2.344766E+02 | global_load_balancing_loss: 9.991755E-01 | loss scale: 1.0 | grad norm: 0.110 | params norm: 11142.254 | params rms: 0.060332 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:14:49.902429 | finish at 2026-10-17 19:38:11 + [2026-10-17 16:27:02] iteration 2333/ 2384 | consumed samples: 74656 | elapsed time per iteration (ms): 220792.8 | throughput per GPU (TFLOP/s/GPU): 24.4 | MFU 7.82% | FLOPs estimator: workload-aware | logical tokens/s: 75986.1 | valid tokens/s: 75986.1 | physical tokens/s: 76468.9 | loss tokens/s: 59069.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.034118E-01 | lm-global-supervised-tokens: 1.304219E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.515930E-02 | lm-token-buffer-mib: 2.328151E+02 | global_load_balancing_loss: 1.000955E+00 | loss scale: 1.0 | grad norm: 0.110 | params norm: 11142.220 | params rms: 0.060331 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:07:40.430755 | finish at 2026-10-17 19:34:42 + [2026-10-17 16:30:49] iteration 2334/ 2384 | consumed samples: 74688 | elapsed time per iteration (ms): 227078.3 | throughput per GPU (TFLOP/s/GPU): 25.9 | MFU 8.30% | FLOPs estimator: workload-aware | logical tokens/s: 73882.9 | valid tokens/s: 73882.9 | physical tokens/s: 74494.5 | loss tokens/s: 51951.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.306235E-01 | lm-global-supervised-tokens: 1.179696E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.837207E-02 | lm-token-buffer-mib: 2.345834E+02 | global_load_balancing_loss: 1.000315E+00 | loss scale: 1.0 | grad norm: 0.226 | params norm: 11142.187 | params rms: 0.060331 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:09:13.915489 | finish at 2026-10-17 19:40:03 + [2026-10-17 16:34:32] iteration 2335/ 2384 | consumed samples: 74720 | elapsed time per iteration (ms): 223108.0 | throughput per GPU (TFLOP/s/GPU): 24.4 | MFU 7.82% | FLOPs estimator: workload-aware | logical tokens/s: 75197.6 | valid tokens/s: 75197.6 | physical tokens/s: 75558.0 | loss tokens/s: 54148.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.969828E-01 | lm-global-supervised-tokens: 1.208089E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.226118E-02 | lm-token-buffer-mib: 2.331871E+02 | global_load_balancing_loss: 9.999257E-01 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11142.150 | params rms: 0.060331 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:02:12.293953 | finish at 2026-10-17 19:36:44 + [2026-10-17 16:38:15] iteration 2336/ 2384 | consumed samples: 74752 | elapsed time per iteration (ms): 223374.2 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.27% | FLOPs estimator: workload-aware | logical tokens/s: 75108.0 | valid tokens/s: 75108.0 | physical tokens/s: 75705.2 | loss tokens/s: 57734.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.837619E-01 | lm-global-supervised-tokens: 1.289633E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.215733E-02 | lm-token-buffer-mib: 2.352695E+02 | global_load_balancing_loss: 9.994578E-01 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11142.115 | params rms: 0.060331 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:58:41.962681 | finish at 2026-10-17 19:36:57 + [2026-10-17 16:41:59] iteration 2337/ 2384 | consumed samples: 74784 | elapsed time per iteration (ms): 223408.2 | throughput per GPU (TFLOP/s/GPU): 24.1 | MFU 7.71% | FLOPs estimator: workload-aware | logical tokens/s: 75096.4 | valid tokens/s: 75096.4 | physical tokens/s: 75861.3 | loss tokens/s: 57395.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.888664E-01 | lm-global-supervised-tokens: 1.282271E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.806698E-02 | lm-token-buffer-mib: 2.354241E+02 | global_load_balancing_loss: 9.999607E-01 | loss scale: 1.0 | grad norm: 0.361 | params norm: 11142.081 | params rms: 0.060331 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:55:00.187743 | finish at 2026-10-17 19:36:59 + [2026-10-17 16:45:43] iteration 2338/ 2384 | consumed samples: 74816 | elapsed time per iteration (ms): 223917.7 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.15% | FLOPs estimator: workload-aware | logical tokens/s: 74925.8 | valid tokens/s: 74925.8 | physical tokens/s: 75420.3 | loss tokens/s: 57730.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.627765E-01 | lm-global-supervised-tokens: 1.292690E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.452816E-02 | lm-token-buffer-mib: 2.343448E+02 | global_load_balancing_loss: 1.001360E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11142.048 | params rms: 0.060331 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:51:40.213248 | finish at 2026-10-17 19:37:23 + [2026-10-17 16:49:25] iteration 2339/ 2384 | consumed samples: 74848 | elapsed time per iteration (ms): 222591.3 | throughput per GPU (TFLOP/s/GPU): 22.5 | MFU 7.22% | FLOPs estimator: workload-aware | logical tokens/s: 75372.2 | valid tokens/s: 75372.2 | physical tokens/s: 75849.6 | loss tokens/s: 54607.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.999203E-01 | lm-global-supervised-tokens: 1.215518E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.255970E-02 | lm-token-buffer-mib: 2.342992E+02 | global_load_balancing_loss: 1.001550E+00 | loss scale: 1.0 | grad norm: 0.199 | params norm: 11142.013 | params rms: 0.060330 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:46:56.606601 | finish at 2026-10-17 19:36:22 + [2026-10-17 16:53:27] iteration 2340/ 2384 | consumed samples: 74880 | elapsed time per iteration (ms): 242071.6 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.23% | FLOPs estimator: workload-aware | logical tokens/s: 69306.7 | valid tokens/s: 69306.7 | physical tokens/s: 69848.3 | loss tokens/s: 50554.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.586672E-01 | lm-global-supervised-tokens: 1.223779E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.947612E-02 | lm-token-buffer-mib: 2.333310E+02 | global_load_balancing_loss: 1.000599E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11141.978 | params rms: 0.060330 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:57:31.151699 | finish at 2026-10-17 19:50:59 + [2026-10-17 16:57:09] iteration 2341/ 2384 | consumed samples: 74912 | elapsed time per iteration (ms): 221548.4 | throughput per GPU (TFLOP/s/GPU): 20.2 | MFU 6.49% | FLOPs estimator: workload-aware | logical tokens/s: 75727.0 | valid tokens/s: 75727.0 | physical tokens/s: 76855.4 | loss tokens/s: 59222.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.167865E-01 | lm-global-supervised-tokens: 1.312058E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.650802E-02 | lm-token-buffer-mib: 2.361855E+02 | global_load_balancing_loss: 1.000565E+00 | loss scale: 1.0 | grad norm: 0.473 | params norm: 11141.942 | params rms: 0.060330 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:38:46.583278 | finish at 2026-10-17 19:35:55 + [2026-10-17 17:00:54] iteration 2342/ 2384 | consumed samples: 74944 | elapsed time per iteration (ms): 225490.2 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.85% | FLOPs estimator: workload-aware | logical tokens/s: 74403.2 | valid tokens/s: 74403.2 | physical tokens/s: 75749.5 | loss tokens/s: 55230.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.144471E-01 | lm-global-supervised-tokens: 1.245383E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.656024E-02 | lm-token-buffer-mib: 2.343094E+02 | global_load_balancing_loss: 1.000049E+00 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11141.908 | params rms: 0.060330 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:37:50.587040 | finish at 2026-10-17 19:38:45 + [2026-10-17 17:04:36] iteration 2343/ 2384 | consumed samples: 74976 | elapsed time per iteration (ms): 221375.3 | throughput per GPU (TFLOP/s/GPU): 21.9 | MFU 7.03% | FLOPs estimator: workload-aware | logical tokens/s: 75786.2 | valid tokens/s: 75786.2 | physical tokens/s: 76175.7 | loss tokens/s: 59283.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.388635E-01 | lm-global-supervised-tokens: 1.312394E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.272259E-02 | lm-token-buffer-mib: 2.334174E+02 | global_load_balancing_loss: 9.992261E-01 | loss scale: 1.0 | grad norm: 0.154 | params norm: 11141.875 | params rms: 0.060330 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:31:16.385499 | finish at 2026-10-17 19:35:52 + [2026-10-17 17:08:17] iteration 2344/ 2384 | consumed samples: 75008 | elapsed time per iteration (ms): 221481.6 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.82% | FLOPs estimator: workload-aware | logical tokens/s: 75749.9 | valid tokens/s: 75749.9 | physical tokens/s: 76234.2 | loss tokens/s: 61025.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.651194E-01 | lm-global-supervised-tokens: 1.351600E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.207645E-02 | lm-token-buffer-mib: 2.333788E+02 | global_load_balancing_loss: 9.984977E-01 | loss scale: 1.0 | grad norm: 0.113 | params norm: 11141.844 | params rms: 0.060329 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:27:39.265194 | finish at 2026-10-17 19:35:57 + [2026-10-17 17:11:59] iteration 2345/ 2384 | consumed samples: 75040 | elapsed time per iteration (ms): 221934.5 | throughput per GPU (TFLOP/s/GPU): 24.4 | MFU 7.83% | FLOPs estimator: workload-aware | logical tokens/s: 75595.2 | valid tokens/s: 75595.2 | physical tokens/s: 76018.7 | loss tokens/s: 53326.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.831084E-01 | lm-global-supervised-tokens: 1.183504E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.653419E-02 | lm-token-buffer-mib: 2.341877E+02 | global_load_balancing_loss: 1.000462E+00 | loss scale: 1.0 | grad norm: 0.115 | params norm: 11141.817 | params rms: 0.060329 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:24:15.445295 | finish at 2026-10-17 19:36:15 + [2026-10-17 17:15:46] iteration 2346/ 2384 | consumed samples: 75072 | elapsed time per iteration (ms): 227001.3 | throughput per GPU (TFLOP/s/GPU): 24.3 | MFU 7.79% | FLOPs estimator: workload-aware | logical tokens/s: 73907.7 | valid tokens/s: 73907.7 | physical tokens/s: 74597.1 | loss tokens/s: 56848.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.957907E-01 | lm-global-supervised-tokens: 1.290466E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.462607E-02 | lm-token-buffer-mib: 2.343245E+02 | global_load_balancing_loss: 1.001882E+00 | loss scale: 1.0 | grad norm: 0.142 | params norm: 11141.789 | params rms: 0.060329 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:23:46.048461 | finish at 2026-10-17 19:39:32 + [2026-10-17 17:19:28] iteration 2347/ 2384 | consumed samples: 75104 | elapsed time per iteration (ms): 221818.9 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.91% | FLOPs estimator: workload-aware | logical tokens/s: 75634.6 | valid tokens/s: 75634.6 | physical tokens/s: 76210.6 | loss tokens/s: 56872.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.993708E-01 | lm-global-supervised-tokens: 1.261539E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.734344E-02 | lm-token-buffer-mib: 2.333898E+02 | global_load_balancing_loss: 1.001345E+00 | loss scale: 1.0 | grad norm: 0.212 | params norm: 11141.763 | params rms: 0.060329 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:16:47.299542 | finish at 2026-10-17 19:36:15 + [2026-10-17 17:23:15] iteration 2348/ 2384 | consumed samples: 75136 | elapsed time per iteration (ms): 226644.5 | throughput per GPU (TFLOP/s/GPU): 24.0 | MFU 7.68% | FLOPs estimator: workload-aware | logical tokens/s: 74024.3 | valid tokens/s: 74024.3 | physical tokens/s: 74776.9 | loss tokens/s: 58299.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.663527E-01 | lm-global-supervised-tokens: 1.321336E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.794252E-02 | lm-token-buffer-mib: 2.350932E+02 | global_load_balancing_loss: 1.001446E+00 | loss scale: 1.0 | grad norm: 0.140 | params norm: 11141.739 | params rms: 0.060329 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:15:59.203219 | finish at 2026-10-17 19:39:14 + [2026-10-17 17:26:56] iteration 2349/ 2384 | consumed samples: 75168 | elapsed time per iteration (ms): 221012.7 | throughput per GPU (TFLOP/s/GPU): 21.6 | MFU 6.94% | FLOPs estimator: workload-aware | logical tokens/s: 75910.6 | valid tokens/s: 75910.6 | physical tokens/s: 76674.8 | loss tokens/s: 54251.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.748840E-01 | lm-global-supervised-tokens: 1.199023E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.141776E-02 | lm-token-buffer-mib: 2.338506E+02 | global_load_balancing_loss: 1.000670E+00 | loss scale: 1.0 | grad norm: 0.180 | params norm: 11141.714 | params rms: 0.060329 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:08:55.445604 | finish at 2026-10-17 19:35:51 + [2026-10-17 17:31:03] iteration 2350/ 2384 | consumed samples: 75200 | elapsed time per iteration (ms): 247514.5 | throughput per GPU (TFLOP/s/GPU): 17.1 | MFU 5.49% | FLOPs estimator: workload-aware | logical tokens/s: 67782.5 | valid tokens/s: 67782.5 | physical tokens/s: 68987.2 | loss tokens/s: 49644.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.581961E-01 | lm-global-supervised-tokens: 1.228780E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.945866E-02 | lm-token-buffer-mib: 2.380613E+02 | global_load_balancing_loss: 9.996662E-01 | loss scale: 1.0 | grad norm: 0.329 | params norm: 11141.691 | params rms: 0.060329 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:20:15.493872 | finish at 2026-10-17 19:51:19 + [2026-10-17 17:34:45] iteration 2351/ 2384 | consumed samples: 75232 | elapsed time per iteration (ms): 222309.2 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.84% | FLOPs estimator: workload-aware | logical tokens/s: 75467.8 | valid tokens/s: 75467.8 | physical tokens/s: 76086.3 | loss tokens/s: 57090.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.769996E-01 | lm-global-supervised-tokens: 1.269182E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.502994E-02 | lm-token-buffer-mib: 2.330350E+02 | global_load_balancing_loss: 9.986107E-01 | loss scale: 1.0 | grad norm: 0.118 | params norm: 11141.669 | params rms: 0.060329 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:02:16.205175 | finish at 2026-10-17 19:37:02 + [2026-10-17 17:38:30] iteration 2352/ 2384 | consumed samples: 75264 | elapsed time per iteration (ms): 224720.7 | throughput per GPU (TFLOP/s/GPU): 22.1 | MFU 7.09% | FLOPs estimator: workload-aware | logical tokens/s: 74657.8 | valid tokens/s: 74657.8 | physical tokens/s: 75497.1 | loss tokens/s: 61809.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.920333E-01 | lm-global-supervised-tokens: 1.388992E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.409005E-02 | lm-token-buffer-mib: 2.358062E+02 | global_load_balancing_loss: 9.996703E-01 | loss scale: 1.0 | grad norm: 0.115 | params norm: 11141.645 | params rms: 0.060328 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:59:51.061249 | finish at 2026-10-17 19:38:21 + [2026-10-17 17:42:09] iteration 2353/ 2384 | consumed samples: 75296 | elapsed time per iteration (ms): 218825.9 | throughput per GPU (TFLOP/s/GPU): 17.5 | MFU 5.61% | FLOPs estimator: workload-aware | logical tokens/s: 76669.2 | valid tokens/s: 76669.2 | physical tokens/s: 77524.7 | loss tokens/s: 59272.1 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.851427E-01 | lm-global-supervised-tokens: 1.297028E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.775635E-02 | lm-token-buffer-mib: 2.352625E+02 | global_load_balancing_loss: 9.986578E-01 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11141.622 | params rms: 0.060328 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:53:03.601875 | finish at 2026-10-17 19:35:13 + [2026-10-17 17:45:53] iteration 2354/ 2384 | consumed samples: 75328 | elapsed time per iteration (ms): 223597.6 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.83% | FLOPs estimator: workload-aware | logical tokens/s: 75032.8 | valid tokens/s: 75032.8 | physical tokens/s: 76319.9 | loss tokens/s: 59633.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.400755E-01 | lm-global-supervised-tokens: 1.333401E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.363826E-02 | lm-token-buffer-mib: 2.353372E+02 | global_load_balancing_loss: 1.000190E+00 | loss scale: 1.0 | grad norm: 0.292 | params norm: 11141.598 | params rms: 0.060328 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:51:47.929387 | finish at 2026-10-17 19:37:41 + [2026-10-17 17:49:35] iteration 2355/ 2384 | consumed samples: 75360 | elapsed time per iteration (ms): 222359.4 | throughput per GPU (TFLOP/s/GPU): 18.8 | MFU 6.02% | FLOPs estimator: workload-aware | logical tokens/s: 75450.8 | valid tokens/s: 75450.8 | physical tokens/s: 76458.6 | loss tokens/s: 55659.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.039707E-01 | lm-global-supervised-tokens: 1.237647E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.710891E-02 | lm-token-buffer-mib: 2.334335E+02 | global_load_balancing_loss: 1.000542E+00 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11141.571 | params rms: 0.060328 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:47:28.421398 | finish at 2026-10-17 19:37:03 + [2026-10-17 17:53:17] iteration 2356/ 2384 | consumed samples: 75392 | elapsed time per iteration (ms): 222072.2 | throughput per GPU (TFLOP/s/GPU): 22.7 | MFU 7.28% | FLOPs estimator: workload-aware | logical tokens/s: 75548.4 | valid tokens/s: 75548.4 | physical tokens/s: 76516.5 | loss tokens/s: 59013.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.484653E-01 | lm-global-supervised-tokens: 1.310529E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.415170E-02 | lm-token-buffer-mib: 2.359273E+02 | global_load_balancing_loss: 1.000899E+00 | loss scale: 1.0 | grad norm: 0.171 | params norm: 11141.546 | params rms: 0.060328 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:43:38.020634 | finish at 2026-10-17 19:36:55 + [2026-10-17 17:56:58] iteration 2357/ 2384 | consumed samples: 75424 | elapsed time per iteration (ms): 221236.8 | throughput per GPU (TFLOP/s/GPU): 21.3 | MFU 6.82% | FLOPs estimator: workload-aware | logical tokens/s: 75833.6 | valid tokens/s: 75833.6 | physical tokens/s: 76667.7 | loss tokens/s: 59592.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.289301E-01 | lm-global-supervised-tokens: 1.318401E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.146232E-02 | lm-token-buffer-mib: 2.342614E+02 | global_load_balancing_loss: 1.000700E+00 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11141.521 | params rms: 0.060328 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:39:33.394307 | finish at 2026-10-17 19:36:32 + [2026-10-17 18:00:41] iteration 2358/ 2384 | consumed samples: 75456 | elapsed time per iteration (ms): 222873.9 | throughput per GPU (TFLOP/s/GPU): 26.0 | MFU 8.32% | FLOPs estimator: workload-aware | logical tokens/s: 75276.5 | valid tokens/s: 75276.5 | physical tokens/s: 75509.9 | loss tokens/s: 52055.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.716033E-01 | lm-global-supervised-tokens: 1.160187E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.109636E-02 | lm-token-buffer-mib: 2.322952E+02 | global_load_balancing_loss: 1.001177E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11141.496 | params rms: 0.060328 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:36:34.720735 | finish at 2026-10-17 19:37:16 + [2026-10-17 18:04:23] iteration 2359/ 2384 | consumed samples: 75488 | elapsed time per iteration (ms): 222207.0 | throughput per GPU (TFLOP/s/GPU): 22.3 | MFU 7.15% | FLOPs estimator: workload-aware | logical tokens/s: 75502.5 | valid tokens/s: 75502.5 | physical tokens/s: 76343.7 | loss tokens/s: 60612.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.254078E-01 | lm-global-supervised-tokens: 1.346850E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.660935E-02 | lm-token-buffer-mib: 2.332932E+02 | global_load_balancing_loss: 1.001201E+00 | loss scale: 1.0 | grad norm: 0.146 | params norm: 11141.467 | params rms: 0.060327 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:32:35.175203 | finish at 2026-10-17 19:36:59 + [2026-10-17 18:08:26] iteration 2360/ 2384 | consumed samples: 75520 | elapsed time per iteration (ms): 242211.7 | throughput per GPU (TFLOP/s/GPU): 19.3 | MFU 6.19% | FLOPs estimator: workload-aware | logical tokens/s: 69266.7 | valid tokens/s: 69266.7 | physical tokens/s: 69669.2 | loss tokens/s: 52957.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.786213E-01 | lm-global-supervised-tokens: 1.282697E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.482358E-02 | lm-token-buffer-mib: 2.338764E+02 | global_load_balancing_loss: 1.000967E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11141.438 | params rms: 0.060327 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:36:53.081491 | finish at 2026-10-17 19:45:19 +times across ranks (ms): + save-checkpoint: + rank 0: 24799.10 + rank 1: 24799.08 + rank 2: 24799.11 + rank 3: 24799.16 + rank 4: 24799.09 + rank 5: 24799.12 + rank 6: 24799.10 + rank 7: 24799.09 + rank 8: 24798.88 + rank 9: 24798.95 + rank 10: 24798.94 + rank 11: 24798.88 + rank 12: 24798.92 + rank 13: 24798.87 + rank 14: 24798.90 + rank 15: 24798.89 + rank 16: 24798.92 + rank 17: 24798.91 + rank 18: 24798.93 + rank 19: 24798.96 + rank 20: 24798.87 + rank 21: 24798.87 + rank 22: 24798.90 + rank 23: 24798.89 + rank 24: 24799.50 + rank 25: 24799.48 + rank 26: 24799.49 + rank 27: 24799.47 + rank 28: 24799.46 + rank 29: 24799.53 + rank 30: 24799.51 + rank 31: 24799.91 + rank 32: 24798.87 + rank 33: 24798.87 + rank 34: 24798.76 + rank 35: 24799.04 + rank 36: 24798.90 + rank 37: 24798.95 + rank 38: 24798.74 + rank 39: 24798.92 + rank 40: 24798.88 + rank 41: 24798.88 + rank 42: 24798.88 + rank 43: 24798.88 + rank 44: 24798.84 + rank 45: 24798.85 + rank 46: 24798.84 + rank 47: 24798.93 + rank 48: 24799.75 + rank 49: 24799.77 + rank 50: 24799.81 + rank 51: 24799.73 + rank 52: 24799.82 + rank 53: 24799.91 + rank 54: 24799.81 + rank 55: 24799.70 + rank 56: 24799.03 + rank 57: 24799.06 + rank 58: 24799.04 + rank 59: 24799.11 + rank 60: 24798.91 + rank 61: 24798.88 + rank 62: 24799.02 + rank 63: 24799.02 + [2026-10-17 18:12:35] iteration 2361/ 2384 | consumed samples: 75552 | elapsed time per iteration (ms): 224369.5 | throughput per GPU (TFLOP/s/GPU): 24.8 | MFU 7.94% | FLOPs estimator: workload-aware | logical tokens/s: 74774.8 | valid tokens/s: 74774.7 | physical tokens/s: 75499.2 | loss tokens/s: 59929.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.361327E-01 | lm-global-supervised-tokens: 1.344639E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 5.125577E-01 | lm-token-buffer-mib: 2.350555E+02 | global_load_balancing_loss: 9.994248E-01 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11141.410 | params rms: 0.060327 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:26:00.497983 | finish at 2026-10-17 19:38:36 + [2026-10-17 18:16:22] iteration 2362/ 2384 | consumed samples: 75584 | elapsed time per iteration (ms): 226642.1 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.40% | FLOPs estimator: workload-aware | logical tokens/s: 74025.1 | valid tokens/s: 74025.1 | physical tokens/s: 74935.6 | loss tokens/s: 55146.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.628219E-01 | lm-global-supervised-tokens: 1.249854E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.591265E-02 | lm-token-buffer-mib: 2.349855E+02 | global_load_balancing_loss: 9.998685E-01 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11141.382 | params rms: 0.060327 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:23:06.125473 | finish at 2026-10-17 19:39:28 + [2026-10-17 18:20:05] iteration 2363/ 2384 | consumed samples: 75616 | elapsed time per iteration (ms): 223370.2 | throughput per GPU (TFLOP/s/GPU): 20.0 | MFU 6.41% | FLOPs estimator: workload-aware | logical tokens/s: 75109.4 | valid tokens/s: 75109.4 | physical tokens/s: 76276.2 | loss tokens/s: 59199.2 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.447251E-01 | lm-global-supervised-tokens: 1.322334E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 6.799983E-02 | lm-token-buffer-mib: 2.375506E+02 | global_load_balancing_loss: 1.000381E+00 | loss scale: 1.0 | grad norm: 0.126 | params norm: 11141.353 | params rms: 0.060327 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:18:10.774829 | finish at 2026-10-17 19:38:16 + [2026-10-17 18:23:47] iteration 2364/ 2384 | consumed samples: 75648 | elapsed time per iteration (ms): 222147.1 | throughput per GPU (TFLOP/s/GPU): 21.4 | MFU 6.85% | FLOPs estimator: workload-aware | logical tokens/s: 75522.8 | valid tokens/s: 75522.8 | physical tokens/s: 77116.5 | loss tokens/s: 57391.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.579518E-01 | lm-global-supervised-tokens: 1.274923E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.821753E-02 | lm-token-buffer-mib: 2.366209E+02 | global_load_balancing_loss: 1.000433E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11141.324 | params rms: 0.060327 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:14:02.941971 | finish at 2026-10-17 19:37:50 + [2026-10-17 18:27:30] iteration 2365/ 2384 | consumed samples: 75680 | elapsed time per iteration (ms): 222410.3 | throughput per GPU (TFLOP/s/GPU): 24.6 | MFU 7.89% | FLOPs estimator: workload-aware | logical tokens/s: 75433.6 | valid tokens/s: 75433.6 | physical tokens/s: 76163.4 | loss tokens/s: 60535.8 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 8.279094E-01 | lm-global-supervised-tokens: 1.346378E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.219713E-02 | lm-token-buffer-mib: 2.345189E+02 | global_load_balancing_loss: 9.998090E-01 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11141.294 | params rms: 0.060326 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:10:25.795166 | finish at 2026-10-17 19:37:55 + [2026-10-17 18:31:13] iteration 2366/ 2384 | consumed samples: 75712 | elapsed time per iteration (ms): 223404.9 | throughput per GPU (TFLOP/s/GPU): 25.6 | MFU 8.20% | FLOPs estimator: workload-aware | logical tokens/s: 75097.6 | valid tokens/s: 75097.6 | physical tokens/s: 75651.6 | loss tokens/s: 52939.6 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.932648E-01 | lm-global-supervised-tokens: 1.182696E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.262020E-02 | lm-token-buffer-mib: 2.344811E+02 | global_load_balancing_loss: 1.000150E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11141.263 | params rms: 0.060326 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:07:01.288274 | finish at 2026-10-17 19:38:14 + [2026-10-17 18:34:57] iteration 2367/ 2384 | consumed samples: 75744 | elapsed time per iteration (ms): 223516.9 | throughput per GPU (TFLOP/s/GPU): 19.7 | MFU 6.31% | FLOPs estimator: workload-aware | logical tokens/s: 75060.0 | valid tokens/s: 75060.0 | physical tokens/s: 76288.0 | loss tokens/s: 55437.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.789155E-01 | lm-global-supervised-tokens: 1.239127E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.915189E-02 | lm-token-buffer-mib: 2.356863E+02 | global_load_balancing_loss: 1.001077E+00 | loss scale: 1.0 | grad norm: 0.102 | params norm: 11141.232 | params rms: 0.060326 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:03:19.787605 | finish at 2026-10-17 19:38:16 + [2026-10-17 18:38:39] iteration 2368/ 2384 | consumed samples: 75776 | elapsed time per iteration (ms): 222327.3 | throughput per GPU (TFLOP/s/GPU): 19.5 | MFU 6.25% | FLOPs estimator: workload-aware | logical tokens/s: 75461.8 | valid tokens/s: 75461.8 | physical tokens/s: 76326.8 | loss tokens/s: 53480.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.165713E-01 | lm-global-supervised-tokens: 1.189025E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.207300E-02 | lm-token-buffer-mib: 2.339821E+02 | global_load_balancing_loss: 1.000660E+00 | loss scale: 1.0 | grad norm: 0.112 | params norm: 11141.203 | params rms: 0.060326 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:59:17.236294 | finish at 2026-10-17 19:37:56 + [2026-10-17 18:42:24] iteration 2369/ 2384 | consumed samples: 75808 | elapsed time per iteration (ms): 225540.6 | throughput per GPU (TFLOP/s/GPU): 22.8 | MFU 7.32% | FLOPs estimator: workload-aware | logical tokens/s: 74386.6 | valid tokens/s: 74386.6 | physical tokens/s: 75739.7 | loss tokens/s: 58015.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.288873E-01 | lm-global-supervised-tokens: 1.308495E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.767612E-02 | lm-token-buffer-mib: 2.349650E+02 | global_load_balancing_loss: 1.000248E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11141.174 | params rms: 0.060326 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:56:23.108997 | finish at 2026-10-17 19:38:47 + [2026-10-17 18:46:28] iteration 2370/ 2384 | consumed samples: 75840 | elapsed time per iteration (ms): 243948.0 | throughput per GPU (TFLOP/s/GPU): 18.1 | MFU 5.79% | FLOPs estimator: workload-aware | logical tokens/s: 68773.7 | valid tokens/s: 68773.7 | physical tokens/s: 69390.3 | loss tokens/s: 51363.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.663929E-01 | lm-global-supervised-tokens: 1.252990E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.379879E-02 | lm-token-buffer-mib: 2.343483E+02 | global_load_balancing_loss: 1.000605E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11141.144 | params rms: 0.060326 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:56:55.272430 | finish at 2026-10-17 19:43:24 + [2026-10-17 18:50:12] iteration 2371/ 2384 | consumed samples: 75872 | elapsed time per iteration (ms): 223477.1 | throughput per GPU (TFLOP/s/GPU): 20.2 | MFU 6.49% | FLOPs estimator: workload-aware | logical tokens/s: 75073.3 | valid tokens/s: 75073.3 | physical tokens/s: 75656.6 | loss tokens/s: 58456.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.204813E-01 | lm-global-supervised-tokens: 1.306358E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.102389E-02 | lm-token-buffer-mib: 2.343282E+02 | global_load_balancing_loss: 1.000207E+00 | loss scale: 1.0 | grad norm: 0.112 | params norm: 11141.112 | params rms: 0.060325 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:48:25.201883 | finish at 2026-10-17 19:38:37 + [2026-10-17 18:53:52] iteration 2372/ 2384 | consumed samples: 75904 | elapsed time per iteration (ms): 220059.1 | throughput per GPU (TFLOP/s/GPU): 22.6 | MFU 7.24% | FLOPs estimator: workload-aware | logical tokens/s: 76239.5 | valid tokens/s: 76239.5 | physical tokens/s: 76630.5 | loss tokens/s: 55175.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.562590E-01 | lm-global-supervised-tokens: 1.214197E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.286157E-02 | lm-token-buffer-mib: 2.326078E+02 | global_load_balancing_loss: 1.000329E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11141.080 | params rms: 0.060325 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:44:00.709622 | finish at 2026-10-17 19:37:53 + [2026-10-17 18:57:40] iteration 2373/ 2384 | consumed samples: 75936 | elapsed time per iteration (ms): 228220.5 | throughput per GPU (TFLOP/s/GPU): 20.2 | MFU 6.47% | FLOPs estimator: workload-aware | logical tokens/s: 73513.1 | valid tokens/s: 73513.1 | physical tokens/s: 74700.3 | loss tokens/s: 51304.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.399871E-01 | lm-global-supervised-tokens: 1.170872E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.655370E-02 | lm-token-buffer-mib: 2.360803E+02 | global_load_balancing_loss: 9.999638E-01 | loss scale: 1.0 | grad norm: 0.164 | params norm: 11141.048 | params rms: 0.060325 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:41:50.425413 | finish at 2026-10-17 19:39:31 + [2026-10-17 19:01:24] iteration 2374/ 2384 | consumed samples: 75968 | elapsed time per iteration (ms): 224377.6 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.61% | FLOPs estimator: workload-aware | logical tokens/s: 74772.2 | valid tokens/s: 74772.2 | physical tokens/s: 75720.4 | loss tokens/s: 55146.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.744265E-01 | lm-global-supervised-tokens: 1.237364E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.214943E-02 | lm-token-buffer-mib: 2.348498E+02 | global_load_balancing_loss: 9.996844E-01 | loss scale: 1.0 | grad norm: 0.137 | params norm: 11141.017 | params rms: 0.060325 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:37:23.776159 | finish at 2026-10-17 19:38:48 + [2026-10-17 19:05:07] iteration 2375/ 2384 | consumed samples: 76000 | elapsed time per iteration (ms): 222736.5 | throughput per GPU (TFLOP/s/GPU): 23.7 | MFU 7.61% | FLOPs estimator: workload-aware | logical tokens/s: 75322.9 | valid tokens/s: 75322.9 | physical tokens/s: 75968.2 | loss tokens/s: 57156.4 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.135043E-01 | lm-global-supervised-tokens: 1.273082E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.584179E-02 | lm-token-buffer-mib: 2.351692E+02 | global_load_balancing_loss: 9.988273E-01 | loss scale: 1.0 | grad norm: 0.165 | params norm: 11140.992 | params rms: 0.060325 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:33:24.628676 | finish at 2026-10-17 19:38:32 + [2026-10-17 19:08:50] iteration 2376/ 2384 | consumed samples: 76032 | elapsed time per iteration (ms): 223176.7 | throughput per GPU (TFLOP/s/GPU): 23.0 | MFU 7.39% | FLOPs estimator: workload-aware | logical tokens/s: 75174.4 | valid tokens/s: 75174.4 | physical tokens/s: 76184.3 | loss tokens/s: 58724.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.124508E-01 | lm-global-supervised-tokens: 1.310589E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.380542E-02 | lm-token-buffer-mib: 2.348383E+02 | global_load_balancing_loss: 1.000534E+00 | loss scale: 1.0 | grad norm: 0.163 | params norm: 11140.970 | params rms: 0.060325 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:29:45.413282 | finish at 2026-10-17 19:38:36 + [2026-10-17 19:12:31] iteration 2377/ 2384 | consumed samples: 76064 | elapsed time per iteration (ms): 220286.1 | throughput per GPU (TFLOP/s/GPU): 22.4 | MFU 7.18% | FLOPs estimator: workload-aware | logical tokens/s: 76160.8 | valid tokens/s: 76160.8 | physical tokens/s: 76841.7 | loss tokens/s: 58416.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.645155E-01 | lm-global-supervised-tokens: 1.286830E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.287969E-02 | lm-token-buffer-mib: 2.333653E+02 | global_load_balancing_loss: 1.001015E+00 | loss scale: 1.0 | grad norm: 0.163 | params norm: 11140.945 | params rms: 0.060325 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:25:42.002916 | finish at 2026-10-17 19:38:13 + [2026-10-17 19:16:12] iteration 2378/ 2384 | consumed samples: 76096 | elapsed time per iteration (ms): 221189.0 | throughput per GPU (TFLOP/s/GPU): 25.6 | MFU 8.19% | FLOPs estimator: workload-aware | logical tokens/s: 75850.0 | valid tokens/s: 75850.0 | physical tokens/s: 76656.0 | loss tokens/s: 57020.9 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 7.077727E-01 | lm-global-supervised-tokens: 1.261239E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.401456E-02 | lm-token-buffer-mib: 2.346887E+02 | global_load_balancing_loss: 1.000139E+00 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11140.918 | params rms: 0.060324 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:22:07.134170 | finish at 2026-10-17 19:38:19 + [2026-10-17 19:19:56] iteration 2379/ 2384 | consumed samples: 76128 | elapsed time per iteration (ms): 223799.1 | throughput per GPU (TFLOP/s/GPU): 22.2 | MFU 7.12% | FLOPs estimator: workload-aware | logical tokens/s: 74965.3 | valid tokens/s: 74965.3 | physical tokens/s: 75526.6 | loss tokens/s: 52820.0 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 4.932884E-01 | lm-global-supervised-tokens: 1.182107E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.224131E-02 | lm-token-buffer-mib: 2.345855E+02 | global_load_balancing_loss: 1.001727E+00 | loss scale: 1.0 | grad norm: 0.141 | params norm: 11140.891 | params rms: 0.060324 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:18:38.995647 | finish at 2026-10-17 19:38:35 + [2026-10-17 19:23:59] iteration 2380/ 2384 | consumed samples: 76160 | elapsed time per iteration (ms): 243583.1 | throughput per GPU (TFLOP/s/GPU): 18.7 | MFU 6.01% | FLOPs estimator: workload-aware | logical tokens/s: 68876.7 | valid tokens/s: 68876.7 | physical tokens/s: 69507.6 | loss tokens/s: 49424.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.449102E-01 | lm-global-supervised-tokens: 1.203897E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.551747E-02 | lm-token-buffer-mib: 2.331971E+02 | global_load_balancing_loss: 1.001267E+00 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11140.864 | params rms: 0.060324 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:16:14.332549 | finish at 2026-10-17 19:40:14 + [2026-10-17 19:27:41] iteration 2381/ 2384 | consumed samples: 76192 | elapsed time per iteration (ms): 221263.9 | throughput per GPU (TFLOP/s/GPU): 24.6 | MFU 7.89% | FLOPs estimator: workload-aware | logical tokens/s: 75824.3 | valid tokens/s: 75824.3 | physical tokens/s: 76587.8 | loss tokens/s: 53987.3 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.975965E-01 | lm-global-supervised-tokens: 1.194543E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.471700E-02 | lm-token-buffer-mib: 2.330941E+02 | global_load_balancing_loss: 9.996806E-01 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11140.835 | params rms: 0.060324 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:11:03.791812 | finish at 2026-10-17 19:38:44 + [2026-10-17 19:31:23] iteration 2382/ 2384 | consumed samples: 76224 | elapsed time per iteration (ms): 222421.5 | throughput per GPU (TFLOP/s/GPU): 19.3 | MFU 6.20% | FLOPs estimator: workload-aware | logical tokens/s: 75429.8 | valid tokens/s: 75429.8 | physical tokens/s: 76188.9 | loss tokens/s: 59967.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 5.787159E-01 | lm-global-supervised-tokens: 1.333811E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.016818E-02 | lm-token-buffer-mib: 2.331569E+02 | global_load_balancing_loss: 9.991398E-01 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11140.802 | params rms: 0.060324 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:07:24.842957 | finish at 2026-10-17 19:38:48 + [2026-10-17 19:35:08] iteration 2383/ 2384 | consumed samples: 76256 | elapsed time per iteration (ms): 225526.5 | throughput per GPU (TFLOP/s/GPU): 24.9 | MFU 7.98% | FLOPs estimator: workload-aware | logical tokens/s: 74391.3 | valid tokens/s: 74391.3 | physical tokens/s: 75295.2 | loss tokens/s: 56502.7 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.765473E-01 | lm-global-supervised-tokens: 1.274286E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 3.795324E-02 | lm-token-buffer-mib: 2.345378E+02 | global_load_balancing_loss: 9.988429E-01 | loss scale: 1.0 | grad norm: 0.123 | params norm: 11140.771 | params rms: 0.060324 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:03:45.526459 | finish at 2026-10-17 19:38:54 + [2026-10-17 19:38:51] iteration 2384/ 2384 | consumed samples: 76288 | elapsed time per iteration (ms): 222158.7 | throughput per GPU (TFLOP/s/GPU): 20.6 | MFU 6.60% | FLOPs estimator: workload-aware | logical tokens/s: 75518.9 | valid tokens/s: 75518.9 | physical tokens/s: 76604.9 | loss tokens/s: 58222.5 | learning rate: 5.000000E-04 | global batch size: 32 | lm loss: 6.096337E-01 | lm-global-supervised-tokens: 1.293464E+07 | lm-global-samples: 3.200000E+01 | lm-token-prepare-seconds: 4.254910E-02 | lm-token-buffer-mib: 2.354543E+02 | global_load_balancing_loss: 9.990426E-01 | loss scale: 1.0 | grad norm: 0.249 | params norm: 11140.739 | params rms: 0.060323 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:00:00 | finish at 2026-10-17 19:38:51 +times across ranks (ms): + evaluate: + rank 0: 630812.31 + rank 1: 630812.31 + rank 2: 630812.25 + rank 3: 630812.25 + rank 4: 630812.25 + rank 5: 630812.25 + rank 6: 630812.25 + rank 7: 630812.31 + rank 8: 630825.69 + rank 9: 630825.69 + rank 10: 630825.69 + rank 11: 630825.69 + rank 12: 630825.69 + rank 13: 630825.69 + rank 14: 630825.69 + rank 15: 630825.75 + rank 16: 630810.38 + rank 17: 630810.44 + rank 18: 630810.44 + rank 19: 630810.38 + rank 20: 630810.44 + rank 21: 630810.44 + rank 22: 630810.44 + rank 23: 630810.44 + rank 24: 630800.12 + rank 25: 630800.12 + rank 26: 630800.12 + rank 27: 630800.12 + rank 28: 630800.12 + rank 29: 630800.19 + rank 30: 630800.12 + rank 31: 630800.12 + rank 32: 630804.38 + rank 33: 630804.38 + rank 34: 630804.31 + rank 35: 630804.38 + rank 36: 630805.31 + rank 37: 630804.25 + rank 38: 630811.69 + rank 39: 630804.44 + rank 40: 630802.38 + rank 41: 630802.38 + rank 42: 630802.38 + rank 43: 630802.31 + rank 44: 630802.38 + rank 45: 630802.38 + rank 46: 630808.56 + rank 47: 630802.38 + rank 48: 630806.44 + rank 49: 630806.44 + rank 50: 630806.44 + rank 51: 630806.50 + rank 52: 630806.50 + rank 53: 630806.50 + rank 54: 630813.88 + rank 55: 630806.50 + rank 56: 630819.12 + rank 57: 630824.44 + rank 58: 630824.38 + rank 59: 630819.12 + rank 60: 630824.06 + rank 61: 630819.06 + rank 62: 630821.19 + rank 63: 630819.12 +------------------------------------------------------------------------------------------------------------------ + validation loss at iteration 2384 on validation set | lm loss value: 1.067154E+00 | lm loss PPL: 2.907094E+00 | +------------------------------------------------------------------------------------------------------------------ diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb/wandb/run-20261011_160551-acr8mx9p/files/wandb-summary.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb/wandb/run-20261011_160551-acr8mx9p/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..af8cd38265d9d4267ef77831a4ca02740d274752 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb/wandb/run-20261011_160551-acr8mx9p/files/wandb-summary.json @@ -0,0 +1 @@ +{"param-absmax/_layer_45_gated_delta::conv1d":0.5546875,"grad-rms/layer_07/moe::shared_expert_fc1":2.2391731553174487e-07,"grad-absmax/layer_44/moe::expert_fc1":2.3807904199202312e-06,"hidden-states-absmax/_layer_53_gated_delta::core_attention":0.6015625,"update-l1/_layer_35_gated_delta::dt_bias":0.00043541385093703866,"grad-absmax/layer_35/moe::shared_experts":1.1615298717515543e-05,"update-rms/_layer_19_gated_delta::A_log":0.00048064025075372194,"hidden-states-mean/_layer_36_gated_delta::query":0.006820400127220169,"hidden-states-rms/_layer_40_input_layernorm":0.6694929967273293,"hidden-states-rms/_layer_25_input_layernorm":0.37463645643888327,"hidden-states-l1/_layer_17_residual_mlp":0.5507683218441393,"hidden-states-rms/_layer_47_attention::o_proj":1.3250642741014664,"update-absmax/_layer_45_gated_delta::conv1d":0.00390625,"param-rms/_layer_21_attention::linear_qkv":0.038336364570743624,"hidden-states-mean/_layer_32_gated_delta::gate":-0.2981952784294222,"hidden-states-mean/_layer_07_residual_mlp":0.0012766246575936268,"hidden-states-rms/_layer_02_gated_delta::query":0.07773389536651819,"hidden-states-mean/_layer_09_gated_delta::value":-0.00041978037099529755,"hidden-states-l1/_layer_43_moe::router":0.010158511251570582,"update-mean/_layer_51_gated_delta::out_proj":1.7965692222787766e-07,"hidden-states-mean/_layer_03_gated_delta::core_attention":7.31328996932238e-05,"update-rms/gated_delta::conv1d":0.0007957401040680186,"hidden-states-rms/_layer_41_residual_mlp":7.867004194575126,"param-mean/_layer_35_gated_delta::conv1d":-0.0006857920670881867,"hidden-states-l1/_layer_11_gated_delta::in_proj":0.8161452320376956,"hidden-states-absmax/_layer_21_attention::core_attention":7.53125,"hidden-states-mean/_layer_24_residual_mlp":-0.009283583786616777,"param-rms/_layer_39_gated_delta::conv1d":0.05768777879136665,"grad-rms/layer_32/gated_delta::A_log":7.23072198452451e-07,"grad-absmax/layer_37/moe::shared_expert_fc2":2.2892868400958832e-06,"update-mean/_layer_56_gated_delta::A_log":-0.00010878767352551222,"param-rms/_layer_21_attention::o_proj":0.03602500891541431,"moe/layer_14/act_rms_min":0.016323043033480644,"hidden-states-rms/_layer_48_moe::router":0.07216878508256834,"hidden-states-rms/_layer_05_gated_delta::beta":1.5751580961717426,"hidden-states-rms/_layer_28_gated_delta::gate":1.2100435692949474,"hidden-states-absmax/_layer_42_gated_delta::key":4.21875,"grad-rms/layer_34/gated_delta::A_log":3.0121706790042095e-06,"hidden-states-rms/_layer_27_residual_attn":3.2744487706596517,"hidden-states-mean/_layer_46_mlp":-0.0064988446718567214,"hidden-states-mean/_layer_55_gated_delta::key":-0.0008972172996437311,"param-rms/_layer_09_gated_delta::out_norm":0.4009177111338049,"hidden-states-rms/_layer_25_moe::router":0.07216878508256834,"hidden-states-rms/_layer_55_moe::shared_experts":0.2998309349272682,"update-rms/_layer_09_gated_delta::A_log":0.0007358157064912878,"grad-absmax/layer_08/moe::shared_expert_fc1":1.240935489477124e-05,"grad-rms/layer_43/moe::shared_expert_fc2":1.6656355335037743e-07,"hidden-states-l1/_layer_16_moe::shared_experts":0.05644165652294833,"hidden-states-absmax/_layer_55_gated_delta::out_norm":196,"param-mean/_layer_49_attention::linear_qkv":9.673200838733464e-05,"update-mean/_layer_00_embedding":-1.194131158399614e-07,"hidden-states-rms/_layer_53_mlp":1.9138740808947396,"hidden-states-absmax/_layer_46_gated_delta::query":7.25,"param-rms/_layer_31_gated_delta::conv1d":0.08783474311264239,"update-l1/_layer_51_gated_delta::dt_bias":0.0004991344176232815,"hidden-states-absmax/_layer_26_gated_delta::key":6.09375,"param-absmax/_layer_27_gated_delta::dt_bias":0.7006890177726746,"update-rms/_layer_25_gated_delta::dt_bias":0.0006475571283953751,"param-absmax/_layer_17_gated_delta::A_log":1.930492877960205,"update-absmax/_layer_26_gated_delta::dt_bias":0.0013910606503486633,"hidden-states-l1/_layer_40_gated_delta::query":0.13639746742218345,"hidden-states-mean/attention::k_layernorm":-0.044771299313631996,"hidden-states-mean/_layer_26_gated_delta::query":0.0024782575827249982,"hidden-states-absmax/_layer_25_gated_delta::out_proj":37.25,"hidden-states-mean/_layer_22_residual_mlp":-0.004758612354381365,"hidden-states-absmax/_layer_20_mlp":63.5,"hidden-states-absmax/_layer_32_input_layernorm":40.25,"hidden-states-mean/_layer_33_gated_delta::gate":-0.5544992154239673,"hidden-states-mean/_layer_06_gated_delta::core_attention":0.00012660889532970133,"hidden-states-l1/attention::k_layernorm":3.0723154801336077,"hidden-states-mean/_layer_26_gated_delta::key":0.008192336232046632,"param-absmax/_layer_02_gated_delta::conv1d":0.357421875,"hidden-states-l1/_layer_41_gated_delta::key":0.10235658018071415,"grad-rms/layer_32/moe::router":4.936187193938905e-07,"hidden-states-rms/_layer_52_residual_mlp":25.25800369895156,"hidden-states-rms/_layer_43_gated_delta::in_proj":1.7268733447439089,"moe/layer_32/act_rms_median":0.1158672347664833,"hidden-states-absmax/moe::router":0.9999935626983643,"hidden-states-mean/_layer_06_residual_attn":0.0017927686958335274,"hidden-states-rms/_layer_36_gated_delta::query":0.10709158434185467,"hidden-states-l1/_layer_04_residual_mlp":0.20893516647042829,"hidden-states-mean/_layer_20_gated_delta::key":0.022832159812111834,"hidden-states-mean/residual_attn":-0.023009365543941507,"param-absmax/_layer_14_gated_delta::dt_bias":0.6883190274238586,"param-mean/_layer_14_gated_delta::A_log":0.609389066696167,"hidden-states-l1/_layer_49_moe::router":0.01025349064723502,"update-mean/_layer_49_attention::linear_qkv":1.4493230082734954e-07,"hidden-states-rms/_layer_42_gated_delta::value":0.05125663991515487,"hidden-states-l1/_layer_30_input_layernorm":0.18301842308264055,"update-mean/_layer_39_gated_delta::conv1d":-1.0823714546859264e-07,"hidden-states-l1/_layer_28_pre_mlp_layernorm":0.1222429495743453,"grad-absmax/layer_21/moe::shared_expert_fc1":1.983172114705667e-05,"hidden-states-mean/_layer_18_pre_mlp_layernorm":-0.000317330218961712,"hidden-states-absmax/_layer_35_gated_delta::core_attention":0.369140625,"param-l1/_layer_21_attention::linear_qkv":0.027560923248529434,"hidden-states-rms/_layer_16_gated_delta::query":0.1324000343517591,"update-mean/_layer_26_gated_delta::A_log":3.905501216650009e-05,"grad-absmax/layer_19/gated_delta::dt_bias":2.2422264009946957e-05,"grad-absmax/layer_09/moe::router":4.559159788186662e-05,"hidden-states-mean/_layer_28_input_layernorm":0.00143119523599091,"update-rms/_layer_28_gated_delta::out_norm":0.001423572239952466,"hidden-states-absmax/_layer_46_gated_delta::value":13.5,"hidden-states-mean/_layer_53_gated_delta::in_proj":-0.01149404377940084,"grad-rms/layer_24/moe::shared_expert_fc2":3.47233574987595e-07,"update-absmax/_layer_42_gated_delta::out_proj":0.001953125,"grad-absmax/layer_35/gated_delta::conv1d":0.0002688367967493832,"hidden-states-absmax/_layer_33_gated_delta::key":5.53125,"param-absmax/_layer_30_gated_delta::A_log":1.672211766242981,"hidden-states-rms/_layer_08_gated_delta::gate":1.0719961664639137,"param-l1/_layer_24_attention::linear_qkv":0.027090933173894882,"update-l1/_layer_02_gated_delta::dt_bias":0.0005685812211595476,"grad-rms/layer_18/moe::expert_fc2":2.2742323435057502e-08,"update-rms/_layer_56_gated_delta::out_norm":0.0009765625,"update-rms/_layer_15_gated_delta::conv1d":0.0008324729779670367,"param-mean/_layer_37_gated_delta::A_log":0.8825088143348694,"hidden-states-absmax/_layer_18_gated_delta::out_proj":9.1875,"hidden-states-rms/_layer_34_gated_delta::key":0.22054149445144866,"param-rms/_layer_02_gated_delta::out_proj":0.04479246833041368,"grad-absmax/layer_53/moe::router":0.00015053754032123834,"grad-absmax/layer_50/moe::shared_experts":3.581307828426361e-05,"update-mean/_layer_45_gated_delta::dt_bias":-0.0001634748186916113,"update-l1/_layer_14_gated_delta::out_proj":0.00014678238949272782,"hidden-states-l1/_layer_41_moe::routed_experts":0.09905111477690229,"hidden-states-rms/_layer_21_pre_mlp_layernorm":0.12999612183425316,"param-absmax/_layer_05_gated_delta::conv1d":0.365234375,"grad-absmax/layer_19/moe::expert_fc1":4.4075063669879455e-06,"hidden-states-rms/_layer_08_gated_delta::g":3.34373023447152,"hidden-states-rms/_layer_07_residual_attn":0.6661116181231056,"grad-rms/layer_37/gated_delta::A_log":1.7745980568215313e-05,"hidden-states-absmax/_layer_52_gated_delta::out_proj":162,"hidden-states-absmax/_layer_48_residual_attn":592,"grad-rms/layer_03/gated_delta::out_proj":2.597345728841637e-07,"update-l1/_layer_11_gated_delta::dt_bias":0.0005125931347720325,"grad-rms/layer_15/moe::shared_expert_fc2":1.8665452767044315e-07,"hidden-states-l1/_layer_14_residual_attn":0.4513817371585056,"hidden-states-mean/_layer_28_gated_delta::g":-7.804312682665906,"update-mean/_layer_38_gated_delta::out_proj":5.729220475814145e-08,"hidden-states-mean/_layer_52_gated_delta::key":-0.038326085457335166,"hidden-states-l1/_layer_32_moe::shared_experts":0.1128051787978624,"hidden-states-rms/residual_attn":9.43406935211575,"update-absmax/_layer_32_gated_delta::out_proj":0.0078125,"update-rms/_layer_56_gated_delta::A_log":0.0007341784548491642,"hidden-states-l1/_layer_10_gated_delta::out_proj":0.08231165496249854,"param-rms/_layer_56_gated_delta::A_log":1.8994289368191697,"hidden-states-rms/_layer_27_pre_mlp_layernorm":0.1688306332388309,"hidden-states-rms/_layer_56_gated_delta::out_proj":5.570974335116232,"hidden-states-mean/_layer_17_moe::router":0.010252138987523553,"hidden-states-mean/_layer_09_moe::shared_experts":-0.00025038965470146,"moe/layer_12/act_rms_min":0.015078740194439888,"hidden-states-mean/_layer_54_gated_delta::g":-5.0253635675148445,"param-mean/_layer_27_gated_delta::conv1d":-0.0005073030479252338,"hidden-states-rms/_layer_17_gated_delta::in_proj":1.2589916546012634,"hidden-states-rms/_layer_06_moe::router":0.07216878508256834,"hidden-states-absmax/_layer_37_gated_delta::query":5.46875,"param-l1/_layer_32_gated_delta::A_log":1.1293935775756836,"hidden-states-l1/_layer_04_gated_delta::value":0.029883649037319306,"update-rms/_layer_27_gated_delta::out_norm":0.0024535829153127173,"hidden-states-rms/_layer_11_gated_delta::gate":1.2454334555008337,"param-l1/_layer_15_gated_delta::A_log":1.1147769689559937,"grad-absmax/layer_05/moe::expert_fc1":9.606193316358258e-07,"hidden-states-l1/_layer_34_gated_delta::in_proj":1.2411575544529247,"hidden-states-mean/_layer_56_gated_delta::out_proj":0.021366654648382027,"hidden-states-l1/_layer_20_moe::router":0.010258031776830637,"hidden-states-rms/_layer_19_moe::router":0.07216878508256834,"hidden-states-l1/_layer_08_input_layernorm":0.16105889213787222,"grad-rms/layer_16/moe::router":3.920285688840863e-07,"param-absmax/_layer_06_gated_delta::out_norm":0.68359375,"param-l1/_layer_53_gated_delta::out_norm":1.4378662109375,"hidden-states-l1/_layer_25_input_layernorm":0.2078281888756977,"param-mean/_layer_07_gated_delta::out_norm":0.22312164306640625,"update-mean/_layer_12_gated_delta::conv1d":-1.603621058166027e-05,"hidden-states-l1/_layer_33_gated_delta::out_norm":0.05827868679507813,"hidden-states-mean/_layer_11_gated_delta::gate":-0.32028970960255976,"hidden-states-rms/_layer_41_moe::shared_experts":0.2802187139321409,"param-l1/_layer_35_gated_delta::dt_bias":0.25861600041389465,"hidden-states-rms/_layer_14_residual_attn":1.1525540087441115,"hidden-states-absmax/_layer_46_moe::routed_experts":70,"hidden-states-l1/_layer_55_gated_delta::out_proj":1.062823779919598,"hidden-states-l1/_layer_54_gated_delta::in_proj":1.4431227171271728,"hidden-states-absmax/_layer_08_gated_delta::out_proj":8.6875,"moe/layer_24/act_rms_max":0.20573227107524872,"grad-rms/layer_21/moe::expert_fc1":1.848029548880782e-08,"hidden-states-l1/_layer_30_moe::router":0.010222445935598363,"update-absmax/_layer_27_gated_delta::dt_bias":0.0017996430397033691,"grad-absmax/layer_35/gated_delta::out_norm":2.2757871192879975e-05,"hidden-states-rms/_layer_08_gated_delta::out_norm":0.03564408592865794,"hidden-states-absmax/_layer_44_residual_mlp":502,"update-absmax/_layer_54_gated_delta::out_norm":0.0078125,"hidden-states-l1/_layer_19_gated_delta::value":0.023429804480648575,"hidden-states-mean/_layer_56_gated_delta::beta":1.6631676749190896,"hidden-states-l1/_layer_23_moe::shared_experts":0.1473141041176443,"hidden-states-rms/_layer_16_gated_delta::in_proj":1.2235234008948155,"hidden-states-mean/_layer_17_gated_delta::query":0.018562745653122145,"grad-absmax/layer_56/gated_delta::dt_bias":3.4794575185514987e-05,"grad-absmax/layer_53/gated_delta::dt_bias":1.9979304852313362e-05,"hidden-states-mean/_layer_32_input_layernorm":0.004239051176792523,"grad-absmax/layer_13/moe::expert_fc2":2.245726591354469e-06,"hidden-states-absmax/_layer_53_gated_delta::g":18.75,"hidden-states-rms/_layer_31_input_layernorm":0.4765294144299181,"moe/layer_34/act_rms_max":0.30614444613456726,"param-rms/_layer_01_gated_delta::out_norm":0.37151553627992084,"hidden-states-l1/_layer_30_residual_mlp":1.6534878794536447,"hidden-states-absmax/_layer_16_gated_delta::beta":12.0625,"hidden-states-absmax/_layer_26_moe::shared_experts":10.9375,"hidden-states-mean/_layer_11_pre_mlp_layernorm":-0.0004844806967429157,"grad-absmax/layer_34/gated_delta::conv1d":0.0003508318914100528,"hidden-states-l1/_layer_35_residual_attn":2.0774109455836944,"update-l1/_layer_40_gated_delta::dt_bias":0.0005350006395019591,"param-mean/mlp::linear_fc2":5.967099241388496e-07,"param-mean/_layer_05_gated_delta::dt_bias":0.30615857243537903,"hidden-states-l1/_layer_37_pre_mlp_layernorm":0.14296700274550947,"hidden-states-absmax/_layer_31_residual_mlp":502,"hidden-states-absmax/_layer_38_gated_delta::core_attention":0.326171875,"hidden-states-l1/_layer_13_residual_attn":0.43637751774519895,"hidden-states-l1/_layer_01_gated_delta::gate":1.891721950974697,"param-rms/_layer_14_gated_delta::conv1d":0.07966856193836006,"hidden-states-l1/_layer_13_residual_mlp":0.44294320856847863,"hidden-states-l1/_layer_36_gated_delta::core_attention":0.0013678442221917776,"hidden-states-mean/_layer_19_input_layernorm":0.004858716970231172,"hidden-states-rms/_layer_10_mlp":0.34124363253092216,"hidden-states-absmax/_layer_53_gated_delta::key":9.3125,"hidden-states-mean/_layer_33_input_layernorm":0.0035320235901503075,"hidden-states-absmax/_layer_03_gated_delta::value":3.015625,"update-rms/_layer_12_gated_delta::out_norm":0.001413066028172635,"grad-rms/layer_16/moe::expert_fc2":2.307395780306904e-08,"update-l1/_layer_29_gated_delta::out_proj":0.00014335691230371594,"hidden-states-absmax/_layer_33_gated_delta::g":18.25,"grad-rms/layer_42/moe::expert_fc1":2.048328172980358e-08,"hidden-states-mean/_layer_51_moe::router":0.010305283987368581,"update-mean/_layer_56_gated_delta::out_proj":-6.195075030746011e-08,"hidden-states-rms/_layer_06_input_layernorm":0.45967731017239394,"hidden-states-l1/_layer_07_mlp":0.0536181282839928,"hidden-states-absmax/_layer_04_moe::shared_experts":3.65625,"param-l1/_layer_20_gated_delta::dt_bias":0.23803779482841492,"param-l1/_layer_39_gated_delta::dt_bias":0.39933907985687256,"hidden-states-l1/_layer_35_moe::shared_experts":0.1411286462698753,"hidden-states-l1/_layer_21_input_layernorm":0.30459777171585223,"hidden-states-l1/_layer_07_gated_delta::core_attention":0.0006414553682378373,"hidden-states-absmax/_layer_24_attention::q_layernorm":19.875,"grad-absmax/layer_37/gated_delta::conv1d":0.0016608722507953644,"hidden-states-absmax/_layer_38_gated_delta::out_norm":68,"moe/layer_07/act_rms_max":0.057736143469810486,"hidden-states-l1/_layer_17_gated_delta::out_proj":0.10040571247407347,"hidden-states-absmax/_layer_23_mlp":29.5,"hidden-states-absmax/_layer_10_gated_delta::out_proj":17.625,"hidden-states-mean/_layer_37_gated_delta::beta":0.17749770611497237,"hidden-states-mean/_layer_18_gated_delta::g":-4.824978449938112,"hidden-states-mean/_layer_29_gated_delta::out_proj":-0.0006620984848110062,"hidden-states-rms/_layer_35_gated_delta::query":0.44621641194391853,"grad-absmax/layer_14/gated_delta::A_log":5.9963804233120754e-06,"hidden-states-l1/_layer_53_gated_delta::in_proj":1.5114582684795492,"hidden-states-mean/_layer_50_mlp":-0.006511018456342037,"hidden-states-l1/_layer_13_attention::k_layernorm":2.2361901087987555,"hidden-states-mean/_layer_06_gated_delta::g":-5.897248494297086,"hidden-states-mean/_layer_26_gated_delta::gate":-0.19769334169546862,"hidden-states-rms/_layer_23_gated_delta::beta":2.1531838063319264,"hidden-states-absmax/_layer_50_attention::k_layernorm":29.25,"hidden-states-absmax/_layer_32_gated_delta::in_proj":47.75,"hidden-states-absmax/_layer_40_gated_delta::value":11.375,"param-absmax/_layer_35_gated_delta::A_log":1.878784418106079,"param-rms/_layer_41_gated_delta::A_log":1.1555092990819644,"grad-rms/layer_02/gated_delta::conv1d":6.802770760022621e-06,"hidden-states-rms/_layer_06_gated_delta::out_norm":0.039347935814392235,"update-l1/_layer_50_attention::linear_qkv":0.00013829943782184273,"param-l1/_layer_18_gated_delta::out_proj":0.03354280814528465,"param-mean/_layer_32_gated_delta::out_norm":0.327484130859375,"update-mean/_layer_02_mlp::linear_fc1":5.2382453930022166e-08,"hidden-states-mean/_layer_21_residual_mlp":-0.003090552888429509,"hidden-states-rms/_layer_22_residual_attn":1.7397511309583589,"param-rms/_layer_34_gated_delta::dt_bias":0.34050212314898853,"hidden-states-absmax/_layer_53_gated_delta::beta":12.875,"hidden-states-l1/_layer_36_residual_mlp":2.2440917782892917,"grad-absmax/layer_22/moe::shared_expert_fc1":2.2063191863708198e-05,"hidden-states-rms/_layer_32_residual_mlp":5.864347198496943,"hidden-states-mean/_layer_39_gated_delta::query":0.018305700591886114,"hidden-states-rms/_layer_38_residual_attn":7.317487707385348,"hidden-states-mean/_layer_38_moe::routed_experts":-0.00021478945479089245,"grad-rms/layer_14/moe::router":4.5251564531142607e-07,"update-l1/_layer_08_gated_delta::dt_bias":0.0007306796032935381,"hidden-states-rms/_layer_11_gated_delta::value":0.06142252626049707,"hidden-states-mean/_layer_30_gated_delta::in_proj":-0.03318490529330268,"param-rms/_layer_52_gated_delta::out_proj":0.04355258295080866,"hidden-states-l1/_layer_35_gated_delta::core_attention":0.001050036409315337,"update-l1/_layer_23_gated_delta::out_norm":0.000579833984375,"hidden-states-absmax/_layer_02_gated_delta::core_attention":0.177734375,"hidden-states-mean/_layer_24_attention::q_layernorm":-0.010535048645442562,"param-rms/attention::o_proj":0.04682500815199212,"update-mean/_layer_06_gated_delta::dt_bias":1.960480585694313e-05,"update-mean/_layer_47_attention::linear_qkv":-1.3680579513675184e-07,"update-rms/_layer_31_gated_delta::conv1d":0.0008645974219541421,"param-absmax/_layer_43_gated_delta::A_log":1.8432501554489136,"param-l1/_layer_56_gated_delta::dt_bias":0.8782429099082947,"hidden-states-l1/_layer_08_gated_delta::out_proj":0.05902016493918478,"update-l1/_layer_33_gated_delta::out_norm":0.0003662109375,"update-absmax/_layer_04_gated_delta::out_norm":0.001953125,"hidden-states-rms/_layer_40_gated_delta::g":7.704129688241013,"grad-rms/layer_16/gated_delta::conv1d":1.9006377070633138e-05,"update-absmax/_layer_36_gated_delta::A_log":0.001241445541381836,"hidden-states-absmax/_layer_26_gated_delta::in_proj":15.1875,"param-rms/_layer_56_gated_delta::out_proj":0.041795913683005836,"grad-rms/layer_07/gated_delta::dt_bias":1.4165045142472718e-06,"param-rms/_layer_03_gated_delta::out_proj":0.04597493148542426,"param-rms/_layer_03_gated_delta::A_log":1.4129131556858878,"hidden-states-mean/_layer_20_gated_delta::gate":0.11616112931503407,"update-absmax/_layer_20_gated_delta::out_proj":0.001953125,"update-l1/_layer_24_attention::linear_qkv":0.00012533411791082472,"param-l1/_layer_18_gated_delta::A_log":0.8845019936561584,"hidden-states-rms/_layer_43_gated_delta::value":0.11680028739068112,"hidden-states-absmax/_layer_49_input_layernorm":43.75,"grad-rms/layer_19/moe::shared_experts":1.0443443526805245e-06,"grad-rms/layer_22/moe::shared_expert_fc2":2.690858653537605e-07,"param-absmax/_layer_40_gated_delta::out_proj":1.1171875,"update-l1/_layer_41_gated_delta::out_norm":0.00042724609375,"update-rms/_layer_14_gated_delta::A_log":0.0005978689939727949,"hidden-states-rms/_layer_06_gated_delta::out_proj":0.08984855987959738,"hidden-states-mean/_layer_29_moe::routed_experts":-0.00012544140092096034,"grad-rms/layer_14/gated_delta::A_log":2.096640863980502e-06,"update-rms/_layer_53_gated_delta::out_norm":0.001760523083722651,"hidden-states-rms/_layer_33_gated_delta::in_proj":1.7207832740642073,"grad-rms/layer_34/moe::expert_fc2":2.2404351939753286e-08,"grad-absmax/layer_46/moe::shared_experts":2.6098421585629694e-05,"hidden-states-absmax/_layer_55_moe::router":0.999454915523529,"param-mean/_layer_52_gated_delta::dt_bias":0.5770559906959534,"hidden-states-l1/gated_delta::out_norm":0.0873455394939657,"update-mean/_layer_35_gated_delta::out_norm":0.000244140625,"hidden-states-absmax/_layer_03_moe::router":0.9768901467323303,"hidden-states-l1/_layer_16_gated_delta::g":5.938640825586417,"grad-rms/layer_23/gated_delta::out_proj":6.166773158464374e-07,"param-mean/_layer_17_gated_delta::out_proj":-1.6203721315832809e-06,"hidden-states-absmax/_layer_56_residual_mlp":2912,"hidden-states-mean/_layer_43_gated_delta::g":-4.509016097666752,"hidden-states-mean/_layer_41_gated_delta::core_attention":9.843507805461064e-06,"hidden-states-l1/_layer_56_pre_mlp_layernorm":0.0856425640752988,"param-rms/_layer_55_gated_delta::dt_bias":0.8757247988281611,"update-absmax/_layer_00_embedding":0.015625,"hidden-states-absmax/_layer_40_gated_delta::query":21.625,"param-rms/_layer_15_gated_delta::dt_bias":0.25352356755726846,"hidden-states-absmax/_layer_27_gated_delta::in_proj":18,"moe/layer_33/act_rms_min_to_median":0.4870456279377277,"hidden-states-absmax/_layer_17_gated_delta::core_attention":0.06591796875,"param-l1/_layer_46_gated_delta::out_proj":0.03173394873738289,"hidden-states-l1/_layer_12_gated_delta::out_proj":0.1222176921031108,"hidden-states-mean/_layer_53_moe::router":0.010322635150573278,"update-l1/_layer_12_gated_delta::A_log":0.00040902686305344105,"grad-rms/layer_29/moe::expert_fc1":2.0146915182162662e-08,"hidden-states-mean/_layer_49_mlp":-0.004464463397349524,"hidden-states-mean/_layer_43_residual_mlp":-0.09200495088159452,"grad-absmax/layer_23/gated_delta::dt_bias":2.073594805551693e-05,"grad-absmax/layer_56/gated_delta::out_proj":0.00026976419030688703,"param-l1/_layer_09_gated_delta::conv1d":0.03877906873822212,"param-absmax/_layer_37_gated_delta::out_norm":2.609375,"hidden-states-rms/_layer_01_gated_delta::value":0.7643879284618367,"hidden-states-rms/_layer_26_gated_delta::query":0.11576974576876668,"hidden-states-absmax/_layer_20_gated_delta::out_proj":20,"hidden-states-rms/_layer_30_pre_mlp_layernorm":0.19043827683601686,"grad-rms/layer_34/moe::shared_experts":1.86419608694353e-06,"hidden-states-absmax/_layer_28_gated_delta::out_proj":70,"grad-rms/layer_18/gated_delta::A_log":4.6534127036886185e-06,"grad-rms/layer_08/gated_delta::out_proj":2.4014777260282896e-07,"update-absmax/_layer_36_gated_delta::conv1d":0.0029296875,"hidden-states-rms/_layer_29_residual_attn":3.85467463698816,"param-mean/_layer_30_gated_delta::A_log":1.0456833839416504,"hidden-states-l1/_layer_45_gated_delta::value":0.0814540706818188,"hidden-states-l1/_layer_22_moe::shared_experts":0.13936488501607,"update-mean/_layer_44_gated_delta::dt_bias":0.00015787547454237938,"update-l1/_layer_07_gated_delta::out_proj":0.00015351998445112258,"param-rms/_layer_15_gated_delta::A_log":1.2401128273823985,"hidden-states-rms/_layer_47_moe::routed_experts":0.3130694658002261,"grad-absmax/layer_06/moe::expert_fc1":1.197377423522994e-05,"param-mean/_layer_39_gated_delta::out_norm":1.06982421875,"grad-rms/layer_31/gated_delta::out_norm":6.51327325731885e-06,"param-l1/_layer_04_gated_delta::out_proj":0.035697754472494125,"hidden-states-absmax/_layer_25_moe::routed_experts":17.5,"param-absmax/mlp::linear_fc2":0.76171875,"hidden-states-l1/gated_delta::g":5.398032033938704,"grad-absmax/layer_55/gated_delta::out_norm":5.654498454532586e-05,"hidden-states-mean/_layer_19_gated_delta::gate":0.06344299029624346,"hidden-states-mean/_layer_36_gated_delta::in_proj":-0.0597156945890916,"hidden-states-mean/_layer_51_pre_mlp_layernorm":-0.0011465762711705292,"hidden-states-l1/_layer_45_gated_delta::core_attention":0.002380717829551438,"hidden-states-mean/_layer_49_input_layernorm":0.012630725853827697,"grad-absmax/layer_20/gated_delta::out_norm":4.029484261991456e-05,"grad-absmax/layer_08/moe::expert_fc1":7.121499265849707e-07,"hidden-states-mean/_layer_46_residual_mlp":-0.09249984085557948,"hidden-states-rms/_layer_08_gated_delta::out_proj":0.087250919204479,"param-absmax/_layer_46_gated_delta::dt_bias":1.0204968452453613,"grad-rms/layer_19/moe::shared_expert_fc1":3.9214699444836654e-07,"hidden-states-l1/_layer_23_gated_delta::in_proj":0.8214356054080545,"update-absmax/_layer_33_gated_delta::dt_bias":0.001589648425579071,"update-mean/_layer_01_mlp::linear_fc1":-7.990637840293857e-08,"hidden-states-mean/_layer_48_gated_delta::in_proj":-0.040331850200874445,"hidden-states-l1/_layer_26_moe::router":0.010278499145865304,"hidden-states-rms/_layer_46_moe::shared_experts":0.7163453314911161,"hidden-states-mean/_layer_42_gated_delta::key":0.01439433716468062,"hidden-states-rms/_layer_18_residual_mlp":1.376167707253936,"hidden-states-mean/_layer_40_gated_delta::beta":-1.6783717487301526,"grad-absmax/layer_19/gated_delta::out_norm":7.270985952345654e-05,"hidden-states-absmax/_layer_03_gated_delta::query":2.828125,"param-l1/_layer_56_gated_delta::out_norm":1.3885498046875,"hidden-states-absmax/_layer_07_gated_delta::out_proj":10.75,"hidden-states-mean/_layer_04_gated_delta::key":0.009767028736117928,"hidden-states-rms/_layer_24_mlp":0.5189551716761441,"hidden-states-mean/_layer_15_input_layernorm":0.00046474529964607057,"update-absmax/_layer_23_gated_delta::A_log":0.0016939640045166016,"param-l1/_layer_43_gated_delta::out_proj":0.03406146168708801,"param-mean/_layer_44_gated_delta::A_log":1.0150612592697144,"grad-rms/layer_16/gated_delta::out_proj":4.5678727905814424e-07,"hidden-states-rms/_layer_55_gated_delta::in_proj":2.6474654004109768,"param-rms/_layer_19_gated_delta::out_norm":0.8510149323938553,"hidden-states-rms/_layer_48_gated_delta::in_proj":1.7122879069392118,"hidden-states-absmax/_layer_14_gated_delta::key":12.0625,"hidden-states-absmax/_layer_46_moe::router":0.9984473586082458,"grad-absmax/layer_03/moe::shared_expert_fc2":1.7809741621022113e-05,"hidden-states-l1/_layer_38_pre_mlp_layernorm":0.14927053740865437,"update-mean/_layer_18_gated_delta::A_log":0.00011770012497436255,"param-rms/_layer_18_gated_delta::out_proj":0.04312699882135544,"hidden-states-rms/_layer_36_mlp":0.40596561965987665,"grad-absmax/layer_06/gated_delta::out_norm":0.0004283915623091161,"hidden-states-absmax/_layer_27_gated_delta::gate":8.6875,"hidden-states-rms/_layer_44_moe::router":0.07216878508256834,"param-mean/_layer_25_gated_delta::dt_bias":-0.1782996505498886,"update-l1/_layer_57_lm_head":0.00019701816199813038,"hidden-states-l1/_layer_50_moe::shared_experts":0.4497741996972799,"param-absmax/_layer_57_lm_head":1.6953125,"grad-absmax/layer_01/gated_delta::out_proj":0.0001473103038733825,"grad-absmax/layer_28/moe::expert_fc2":1.096124265131948e-06,"hidden-states-rms/_layer_19_gated_delta::query":0.10777459535069481,"grad-absmax/layer_28/moe::shared_expert_fc1":2.9299617381184362e-05,"hidden-states-mean/_layer_25_moe::shared_experts":-0.0018326059072963485,"grad-absmax/layer_08/gated_delta::out_norm":2.8539372578961775e-05,"grad-rms/layer_34/moe::router":4.26874418042892e-07,"grad-absmax/layer_05/moe::router":7.500187348341569e-05,"hidden-states-l1/_layer_31_moe::shared_experts":0.11355918509473623,"hidden-states-l1/_layer_10_mlp":0.0695477136430399,"hidden-states-rms/_layer_56_gated_delta::out_norm":1.2393610983426215,"param-rms/_layer_40_gated_delta::out_proj":0.04359758209618342,"hidden-states-rms/_layer_08_pre_mlp_layernorm":0.08307485612463891,"update-mean/_layer_52_gated_delta::dt_bias":0.00019588321447372437,"hidden-states-mean/_layer_53_gated_delta::key":-4.585035459030921e-05,"hidden-states-l1/_layer_50_moe::router":0.010297616096952136,"update-l1/_layer_36_gated_delta::dt_bias":0.00042500393465161324,"hidden-states-l1/_layer_49_moe::shared_experts":0.3362832470277842,"hidden-states-rms/_layer_53_pre_mlp_layernorm":0.23196656421917022,"update-absmax/_layer_24_attention::linear_qkv":0.001953125,"hidden-states-rms/_layer_27_moe::router":0.07216878508256834,"update-rms/_layer_40_gated_delta::conv1d":0.0008165774379968558,"grad-rms/layer_02/gated_delta::out_proj":2.538163444191911e-07,"grad-rms/layer_30/moe::shared_expert_fc1":2.5925369210422835e-07,"hidden-states-rms/_layer_09_gated_delta::value":0.07419455726629773,"update-l1/_layer_43_gated_delta::conv1d":0.0006059007137082517,"grad-rms/layer_53/moe::shared_experts":3.7304523027177923e-06,"hidden-states-absmax/_layer_37_gated_delta::value":6.34375,"grad-rms/layer_43/gated_delta::A_log":2.965330053822677e-06,"param-mean/_layer_02_gated_delta::dt_bias":0.28381088376045227,"moe/layer_25/act_rms_min_to_median":0.3496744817806279,"hidden-states-l1/_layer_18_mlp":0.10782363806052661,"update-mean/_layer_25_gated_delta::dt_bias":1.1779280612245202e-05,"hidden-states-rms/_layer_34_gated_delta::in_proj":1.8266245394383545,"update-mean/_layer_06_gated_delta::out_norm":5.340576171875e-05,"hidden-states-rms/_layer_16_mlp":0.1378768590955809,"update-l1/_layer_32_gated_delta::conv1d":0.0006628101691603661,"grad-absmax/layer_49/moe::router":6.497385766124353e-05,"grad-absmax/layer_28/gated_delta::out_norm":5.551068170461804e-05,"update-absmax/_layer_01_gated_delta::conv1d":0.003173828125,"grad-absmax/layer_29/gated_delta::out_norm":6.755986396456137e-05,"hidden-states-rms/_layer_43_gated_delta::g":4.919597282801743,"hidden-states-l1/_layer_23_mlp":0.2140494244157582,"param-absmax/_layer_15_gated_delta::dt_bias":0.5409598350524902,"update-rms/_layer_29_gated_delta::conv1d":0.0008196204859508849,"grad-rms/layer_07/moe::router":3.212044227480749e-07,"param-rms/_layer_54_gated_delta::out_proj":0.04264866482449959,"hidden-states-mean/_layer_49_moe::routed_experts":-0.0005637265195711618,"hidden-states-rms/_layer_47_moe::router":0.07216878508256834,"hidden-states-absmax/_layer_51_gated_delta::gate":113,"param-absmax/_layer_02_gated_delta::A_log":2.5536789894104004,"hidden-states-absmax/_layer_04_gated_delta::g":19.5,"hidden-states-mean/_layer_07_gated_delta::out_proj":-0.0001256776745074823,"grad-rms/layer_34/gated_delta::out_proj":4.017946940292951e-07,"update-absmax/_layer_39_gated_delta::conv1d":0.0029449462890625,"hidden-states-mean/_layer_05_gated_delta::out_proj":9.598995946507418e-05,"hidden-states-absmax/_layer_35_moe::router":0.999306857585907,"hidden-states-rms/_layer_16_gated_delta::beta":2.348771616218785,"param-mean/_layer_30_gated_delta::out_proj":-2.120063072652556e-05,"param-absmax/_layer_28_gated_delta::out_proj":1.015625,"hidden-states-absmax/_layer_11_gated_delta::key":6.0625,"hidden-states-rms/_layer_31_gated_delta::beta":2.016332696258116,"hidden-states-absmax/_layer_44_input_layernorm":30,"hidden-states-mean/_layer_38_mlp":-0.002597353730780331,"grad-rms/layer_39/moe::shared_expert_fc2":1.3326641423763762e-07,"hidden-states-mean/_layer_08_moe::shared_experts":-0.00014615121759228023,"param-mean/_layer_19_gated_delta::out_norm":0.808441162109375,"update-mean/_layer_14_gated_delta::dt_bias":-0.00012401143612805754,"update-rms/_layer_09_gated_delta::dt_bias":0.000733688774981062,"hidden-states-absmax/_layer_39_gated_delta::out_norm":83,"grad-absmax/layer_03/gated_delta::out_norm":0.00017509324243292212,"grad-absmax/layer_36/moe::shared_experts":1.0545906661718618e-05,"hidden-states-l1/_layer_06_mlp":0.04859980775115583,"grad-rms/layer_20/moe::expert_fc1":1.7706844930225182e-08,"hidden-states-absmax/_layer_18_mlp":9.875,"update-absmax/_layer_20_gated_delta::conv1d":0.00390625,"param-absmax/_layer_56_gated_delta::dt_bias":1.2747522592544556,"hidden-states-mean/_layer_13_moe::shared_experts":0.0005067795886358128,"param-l1/_layer_45_gated_delta::dt_bias":0.4519028961658478,"hidden-states-mean/_layer_04_gated_delta::value":-0.0019084240722267714,"grad-absmax/layer_43/moe::router":3.7755595258204266e-05,"grad-absmax/layer_53/moe::expert_fc2":2.98462532555277e-06,"param-mean/_layer_54_gated_delta::A_log":1.3781037330627441,"hidden-states-absmax/_layer_56_moe::shared_experts":560,"hidden-states-mean/_layer_04_gated_delta::beta":-0.2845858078701267,"grad-absmax/layer_29/moe::shared_expert_fc1":2.7347554350853898e-05,"hidden-states-mean/_layer_10_gated_delta::core_attention":6.644576997967303e-05,"hidden-states-l1/_layer_33_gated_delta::value":0.05848657291967546,"update-mean/_layer_10_gated_delta::out_norm":-0.0002288818359375,"hidden-states-mean/_layer_28_gated_delta::beta":-2.5300772821995356,"hidden-states-l1/_layer_22_mlp":0.20683979473588404,"grad-rms/layer_19/moe::expert_fc2":2.2671086255332718e-08,"grad-rms/layer_12/gated_delta::out_norm":3.777686829375117e-05,"hidden-states-rms/_layer_24_moe::router":0.07216878508256834,"grad-rms/layer_44/gated_delta::dt_bias":1.0251358258663772e-05,"hidden-states-mean/_layer_19_moe::shared_experts":0.0004993451306747055,"hidden-states-absmax/_layer_09_gated_delta::key":3.140625,"hidden-states-absmax/_layer_48_gated_delta::value":3.53125,"update-mean/_layer_14_gated_delta::conv1d":8.669022463436704e-06,"update-mean/_layer_02_gated_delta::out_norm":7.62939453125e-05,"param-rms/_layer_39_gated_delta::out_proj":0.04388529732570641,"grad-rms/layer_01/mlp::linear_fc1":1.2390775836739405e-07,"hidden-states-l1/_layer_53_gated_delta::query":0.08104073477276978,"update-mean/_layer_04_gated_delta::dt_bias":-1.0466668754816055e-05,"hidden-states-rms/_layer_14_moe::router":0.07216878508256834,"param-mean/_layer_14_gated_delta::out_norm":0.3956756591796875,"hidden-states-mean/_layer_17_gated_delta::in_proj":-0.056682321212422046,"grad-absmax/layer_27/moe::shared_experts":4.770471605297644e-06,"update-rms/_layer_17_gated_delta::conv1d":0.0008239997202084564,"update-rms/_layer_41_gated_delta::out_proj":0.0002039444148298337,"hidden-states-absmax/_layer_52_gated_delta::in_proj":78.5,"update-rms/_layer_55_gated_delta::out_norm":0.0021836601342771385,"param-rms/_layer_19_gated_delta::conv1d":0.06390654476979181,"hidden-states-l1/_layer_07_gated_delta::gate":0.7954781407687832,"hidden-states-rms/_layer_52_gated_delta::gate":2.175633443839049,"hidden-states-rms/_layer_42_input_layernorm":0.4441363811383603,"update-absmax/_layer_03_gated_delta::dt_bias":0.0012725889682769775,"hidden-states-mean/_layer_28_gated_delta::out_proj":-0.0007454211315269627,"hidden-states-l1/_layer_28_moe::shared_experts":0.08925761601419639,"update-rms/_layer_44_gated_delta::dt_bias":0.0006882699239176052,"param-mean/gated_delta::dt_bias":0.28433122181770754,"grad-rms/layer_53/gated_delta::dt_bias":3.826572924709351e-06,"hidden-states-mean/_layer_56_residual_attn":0.09197916441907458,"grad-rms/layer_30/moe::shared_expert_fc2":3.6901175306237656e-07,"hidden-states-l1/_layer_38_gated_delta::beta":1.139522926750532,"grad-absmax/layer_41/gated_delta::A_log":9.2971560661681e-05,"grad-absmax/layer_51/moe::shared_experts":1.6996576960082166e-05,"grad-rms/layer_01/gated_delta::out_norm":2.410737563735768e-05,"param-absmax/_layer_33_gated_delta::conv1d":0.6015625,"moe/layer_08/act_rms_median":0.032061271369457245,"grad-rms/layer_32/gated_delta::conv1d":6.5987612600432486e-06,"hidden-states-rms/_layer_41_gated_delta::in_proj":1.583715515655013,"hidden-states-mean/_layer_53_input_layernorm":0.0064698828646401006,"update-mean/_layer_44_gated_delta::out_proj":-1.6805259406282858e-08,"hidden-states-rms/_layer_45_input_layernorm":0.7756916376007318,"hidden-states-mean/_layer_18_gated_delta::out_proj":-0.00026138278395589103,"hidden-states-l1/_layer_48_gated_delta::out_proj":0.7774138185279154,"hidden-states-l1/_layer_48_gated_delta::value":0.03907436598233771,"hidden-states-absmax/_layer_52_gated_delta::out_norm":194,"grad-rms/layer_22/moe::router":5.635734225330124e-07,"moe/layer_01/act_rms_median":0,"param-absmax/_layer_52_gated_delta::out_proj":1.3515625,"moe/layer_40/act_rms_min_to_median":0.5420221329689118,"hidden-states-l1/_layer_22_pre_mlp_layernorm":0.1164294247543126,"hidden-states-l1/_layer_16_residual_attn":0.5051963931556217,"grad-rms/layer_19/gated_delta::A_log":8.118835833063138e-06,"hidden-states-absmax/_layer_12_gated_delta::core_attention":0.2041015625,"param-rms/_layer_46_gated_delta::A_log":1.2866483334403003,"hidden-states-rms/_layer_07_gated_delta::core_attention":0.001411757646021018,"grad-rms/layer_34/gated_delta::conv1d":1.1877941146793348e-05,"param-rms/_layer_25_gated_delta::conv1d":0.0916450137154975,"update-rms/mlp::linear_fc1":0.00021777159573343308,"hidden-states-absmax/_layer_18_gated_delta::out_norm":15.25,"hidden-states-absmax/_layer_40_moe::shared_experts":28.125,"hidden-states-absmax/_layer_08_gated_delta::out_norm":12.75,"update-absmax/embedding":0.015625,"param-rms/_layer_48_gated_delta::out_norm":1.5785953840473639,"hidden-states-absmax/_layer_09_moe::shared_experts":5.90625,"param-l1/_layer_40_gated_delta::out_norm":0.722320556640625,"grad-absmax/layer_15/moe::router":7.549105794169009e-05,"hidden-states-absmax/_layer_48_gated_delta::query":3.921875,"update-l1/_layer_56_gated_delta::A_log":0.0006184744997881353,"update-l1/_layer_15_gated_delta::out_norm":0.000640869140625,"hidden-states-mean/_layer_35_gated_delta::core_attention":0.0003247097288843666,"hidden-states-l1/_layer_50_input_layernorm":0.3137765900822389,"grad-rms/layer_15/moe::shared_expert_fc1":3.2545251537864613e-07,"hidden-states-mean/_layer_04_gated_delta::in_proj":-0.13133059667239688,"hidden-states-mean/_layer_06_gated_delta::gate":-0.4215441848688346,"hidden-states-rms/_layer_45_mlp":0.7450653942198854,"hidden-states-absmax/_layer_42_pre_mlp_layernorm":3.875,"moe/layer_54/act_rms_median":0.09004148840904236,"update-absmax/_layer_25_gated_delta::out_proj":0.00390625,"hidden-states-rms/_layer_10_gated_delta::gate":1.084215578762233,"grad-absmax/layer_25/gated_delta::conv1d":0.00028613911126740277,"hidden-states-mean/_layer_27_gated_delta::gate":0.13121773376676246,"update-mean/_layer_05_gated_delta::A_log":-9.730644524097443e-05,"hidden-states-mean/_layer_48_gated_delta::value":0.007852755491764007,"param-mean/_layer_07_gated_delta::dt_bias":0.22585317492485046,"hidden-states-rms/_layer_20_moe::routed_experts":0.060353619361577376,"hidden-states-absmax/_layer_04_gated_delta::key":2.203125,"hidden-states-rms/_layer_45_gated_delta::beta":2.652444402961027,"hidden-states-mean/_layer_46_gated_delta::in_proj":-0.03412466017401991,"hidden-states-l1/_layer_24_residual_attn":1.0250288277898822,"update-mean/_layer_04_gated_delta::out_norm":-4.57763671875e-05,"hidden-states-rms/_layer_03_residual_mlp":0.5507558509253544,"hidden-states-l1/_layer_14_gated_delta::in_proj":0.7703841664240487,"hidden-states-mean/_layer_34_moe::routed_experts":-0.0001779949396492144,"param-mean/_layer_16_gated_delta::conv1d":-0.001067125704139471,"grad-absmax/layer_15/gated_delta::conv1d":0.00044974684715270996,"hidden-states-l1/_layer_45_gated_delta::in_proj":1.6400123539689009,"hidden-states-absmax/_layer_33_gated_delta::beta":8.5,"hidden-states-l1/_layer_21_pre_mlp_layernorm":0.09670224448469215,"update-absmax/_layer_02_gated_delta::A_log":0.001703500747680664,"update-rms/_layer_10_gated_delta::out_proj":0.00021488339972821304,"hidden-states-rms/_layer_29_gated_delta::core_attention":0.005334586099086146,"hidden-states-absmax/_layer_25_gated_delta::out_norm":25.25,"grad-rms/layer_24/moe::shared_expert_fc1":4.383533169148536e-07,"moe/layer_20/act_rms_median":0.06404299288988113,"param-absmax/_layer_34_gated_delta::out_proj":0.62890625,"grad-rms/layer_13/attention::o_proj":1.2210864968211576e-06,"param-rms/_layer_07_gated_delta::conv1d":0.05933562544865576,"hidden-states-rms/_layer_12_gated_delta::core_attention":0.0052593283586414435,"hidden-states-absmax/_layer_48_gated_delta::g":20,"hidden-states-rms/_layer_05_residual_mlp":0.6147860031407455,"param-l1/_layer_36_gated_delta::A_log":1.5241872072219849,"hidden-states-l1/_layer_56_input_layernorm":0.34592766713233797,"hidden-states-absmax/_layer_55_gated_delta::value":9.625,"grad-absmax/layer_16/moe::router":3.0058126867515966e-05,"hidden-states-absmax/_layer_27_gated_delta::g":18,"update-mean/_layer_05_gated_delta::out_norm":3.0517578125e-05,"hidden-states-mean/_layer_36_pre_mlp_layernorm":0.00018117357808185555,"hidden-states-mean/_layer_27_gated_delta::core_attention":0.00013810905702687734,"hidden-states-absmax/_layer_38_gated_delta::query":6.125,"load_balancing_loss":0,"hidden-states-mean/_layer_53_gated_delta::out_norm":-0.0031718839015822294,"update-mean/_layer_52_gated_delta::out_proj":-2.9289140002219938e-08,"update-mean/_layer_09_gated_delta::dt_bias":-0.0003503304615151137,"param-rms/_layer_30_gated_delta::dt_bias":0.17337241605519085,"grad-rms/layer_38/gated_delta::out_norm":7.366681737126664e-06,"hidden-states-l1/_layer_39_gated_delta::key":0.05691007830620746,"grad-rms/layer_53/gated_delta::A_log":3.83669216735381e-06,"grad-rms/layer_04/gated_delta::conv1d":1.099074453451601e-05,"hidden-states-l1/_layer_56_mlp":0.47254448801809407,"grad-absmax/layer_11/gated_delta::dt_bias":5.269902430882212e-06,"param-l1/_layer_19_gated_delta::out_norm":0.808441162109375,"grad-absmax/layer_42/moe::expert_fc2":2.542266884120181e-06,"hidden-states-rms/_layer_39_moe::routed_experts":0.1660724615000554,"param-l1/_layer_07_gated_delta::dt_bias":0.2686725854873657,"hidden-states-l1/_layer_34_mlp":0.2259044618492677,"param-absmax/_layer_55_gated_delta::conv1d":0.349609375,"param-rms/_layer_25_gated_delta::out_norm":0.7588612879251193,"hidden-states-rms/_layer_55_gated_delta::out_proj":2.4805201284053857,"hidden-states-mean/_layer_01_gated_delta::g":-7.419745376079724,"grad-rms/layer_35/gated_delta::out_norm":7.592805132333411e-06,"hidden-states-rms/_layer_30_residual_mlp":4.96384125847456,"hidden-states-mean/_layer_18_input_layernorm":0.0014666810750357598,"hidden-states-l1/_layer_13_input_layernorm":0.23755788654779036,"update-l1/_layer_08_gated_delta::out_norm":0.000579833984375,"param-l1/_layer_00_embedding":0.28660061955451965,"hidden-states-mean/_layer_03_residual_attn":0.0010088830060670383,"hidden-states-rms/_layer_09_residual_mlp":0.758568461429859,"hidden-states-absmax/_layer_23_residual_attn":500,"hidden-states-absmax/_layer_38_gated_delta::out_proj":20.5,"param-absmax/_layer_07_gated_delta::out_norm":0.466796875,"hidden-states-rms/_layer_30_moe::shared_experts":0.6763243026788042,"hidden-states-absmax/_layer_10_gated_delta::out_norm":17,"update-rms/_layer_06_gated_delta::dt_bias":0.0005014309742709276,"param-mean/_layer_03_gated_delta::out_norm":0.13102340698242188,"hidden-states-mean/_layer_33_moe::routed_experts":-0.0002596339379068121,"update-rms/_layer_46_gated_delta::dt_bias":0.0005536598676943223,"hidden-states-rms/_layer_51_moe::router":0.07216878508256834,"param-mean/_layer_15_gated_delta::dt_bias":0.03756916522979736,"update-rms/_layer_10_gated_delta::A_log":0.0004978783934406567,"hidden-states-mean/_layer_19_gated_delta::in_proj":-0.037187900631553805,"hidden-states-l1/_layer_23_input_layernorm":0.2011194170855781,"param-rms/lm_head":0.05594011503772113,"hidden-states-absmax/_layer_23_pre_mlp_layernorm":6.21875,"grad-absmax/layer_36/gated_delta::out_proj":0.00021568084775935858,"hidden-states-mean/_layer_12_gated_delta::gate":-0.1110519079627881,"grad-rms/layer_27/gated_delta::out_norm":1.1917161159828622e-05,"param-absmax/_layer_15_gated_delta::conv1d":0.72265625,"hidden-states-rms/_layer_23_mlp":0.33341917408906957,"hidden-states-absmax/_layer_11_gated_delta::query":3.78125,"hidden-states-absmax/_layer_55_pre_mlp_layernorm":10.1875,"update-l1/_layer_05_gated_delta::out_proj":0.000154815090354532,"param-l1/_layer_16_gated_delta::dt_bias":0.12249226123094559,"param-absmax/_layer_54_gated_delta::out_norm":1.921875,"hidden-states-mean/_layer_44_gated_delta::out_norm":-0.004072508328856031,"hidden-states-l1/_layer_12_gated_delta::value":0.03885210396076488,"hidden-states-absmax/_layer_54_gated_delta::value":10.125,"hidden-states-rms/_layer_04_gated_delta::value":0.05264063605320766,"hidden-states-mean/_layer_40_residual_attn":-0.08591509889544235,"hidden-states-l1/_layer_38_gated_delta::query":0.05391593920182457,"hidden-states-mean/_layer_44_gated_delta::in_proj":-0.07032583451690999,"hidden-states-rms/_layer_07_moe::shared_experts":0.04518606680432133,"hidden-states-rms/_layer_56_gated_delta::key":0.14561625003276232,"param-absmax/_layer_37_gated_delta::A_log":1.9073984622955322,"grad-absmax/layer_09/gated_delta::out_proj":4.256120519130491e-05,"hidden-states-rms/_layer_40_gated_delta::out_norm":0.25521370236481067,"hidden-states-mean/_layer_14_mlp":0.0002444969141349243,"hidden-states-rms/_layer_13_attention::k_layernorm":3.067902634318951,"hidden-states-rms/_layer_03_gated_delta::gate":0.95527733673846,"grad-absmax/layer_40/moe::shared_experts":7.341344826272689e-06,"grad-absmax/layer_14/gated_delta::conv1d":0.00028221067623235285,"hidden-states-rms/_layer_48_gated_delta::query":0.14851978437974528,"hidden-states-mean/_layer_17_gated_delta::core_attention":9.639905750665754e-05,"grad-absmax/layer_40/gated_delta::out_proj":5.73378820263315e-05,"update-mean/_layer_08_gated_delta::A_log":-0.0003154939040541649,"hidden-states-l1/_layer_07_residual_mlp":0.261679263315113,"hidden-states-l1/_layer_30_gated_delta::value":0.030020370932861678,"param-rms/_layer_30_gated_delta::conv1d":0.07471832489218211,"hidden-states-l1/_layer_52_gated_delta::value":0.1189308680809579,"update-mean/_layer_52_gated_delta::conv1d":-3.2250682124868035e-06,"hidden-states-rms/_layer_30_gated_delta::g":5.346204367850507,"hidden-states-mean/_layer_14_gated_delta::core_attention":4.255535675639814e-05,"update-l1/_layer_06_gated_delta::out_proj":0.0001509663270553574,"hidden-states-absmax/_layer_23_residual_mlp":500,"hidden-states-rms/_layer_34_moe::shared_experts":0.18861741785067063,"grad-absmax/layer_09/moe::expert_fc1":6.930300173735304e-07,"update-rms/_layer_46_gated_delta::out_proj":0.00019274810461628655,"hidden-states-rms/_layer_04_pre_mlp_layernorm":0.0534171641813492,"hidden-states-mean/_layer_14_moe::router":0.01024958530061806,"update-l1/_layer_06_gated_delta::A_log":0.0003907903446815908,"update-rms/_layer_09_gated_delta::out_proj":0.00021465333259715645,"moe/layer_48/act_rms_max":0.6579798460006714,"update-mean/_layer_27_gated_delta::out_norm":0.000152587890625,"hidden-states-absmax/_layer_07_gated_delta::core_attention":0.1572265625,"hidden-states-mean/_layer_37_gated_delta::g":-2.7218302156722802,"grad-rms/layer_25/gated_delta::out_norm":1.2274705472126637e-05,"grad-absmax/layer_48/gated_delta::out_proj":7.252148498082533e-05,"hidden-states-absmax/_layer_50_attention::core_attention":20.75,"hidden-states-l1/_layer_45_gated_delta::out_norm":0.12083968470569667,"update-mean/_layer_34_gated_delta::out_proj":8.043380717026594e-08,"param-l1/_layer_22_attention::o_proj":0.029901865869760513,"grad-absmax/layer_56/moe::router":8.3631653978955e-05,"update-absmax/_layer_11_gated_delta::dt_bias":0.001769721508026123,"hidden-states-rms/_layer_18_gated_delta::out_proj":0.1609951104758228,"update-mean/_layer_28_gated_delta::out_proj":2.0848042936449929e-07,"hidden-states-rms/_layer_29_input_layernorm":0.5551024277285553,"hidden-states-rms/_layer_30_gated_delta::beta":1.4683678375797562,"hidden-states-absmax/_layer_29_moe::router":0.9998725652694702,"hidden-states-absmax/_layer_19_gated_delta::core_attention":0.083984375,"hidden-states-mean/_layer_25_moe::router":0.010255072654099886,"grad-rms/layer_03/gated_delta::dt_bias":5.847749825434564e-06,"hidden-states-rms/_layer_40_mlp":0.39480406227266085,"hidden-states-mean/_layer_31_gated_delta::value":0.0034848751463592226,"hidden-states-mean/_layer_12_moe::shared_experts":-0.0005892328456247265,"hidden-states-l1/gated_delta::beta":1.7355618782007745,"grad-absmax/layer_39/gated_delta::out_norm":2.1131476387381554e-05,"hidden-states-mean/_layer_52_gated_delta::value":0.009770782897863733,"hidden-states-rms/_layer_39_gated_delta::value":0.07535231119971114,"hidden-states-mean/_layer_48_moe::routed_experts":-0.00037960459743394814,"param-l1/_layer_35_gated_delta::out_norm":0.842559814453125,"hidden-states-absmax/_layer_47_pre_mlp_layernorm":6.84375,"hidden-states-mean/_layer_37_moe::routed_experts":-0.00025116539453196165,"grad-absmax/layer_54/gated_delta::out_proj":5.542901635635644e-05,"hidden-states-rms/_layer_41_gated_delta::query":0.47539690099784226,"grad-absmax/layer_31/moe::shared_expert_fc2":5.655411769112106e-06,"grad-absmax/layer_12/moe::expert_fc2":4.67262725578621e-06,"grad-rms/layer_41/gated_delta::dt_bias":1.5505432440280235e-05,"hidden-states-absmax/_layer_05_gated_delta::value":3.875,"hidden-states-absmax/_layer_12_moe::shared_experts":46.75,"hidden-states-mean/_layer_28_moe::shared_experts":-0.000796042361197455,"hidden-states-absmax/_layer_27_residual_mlp":502,"hidden-states-mean/_layer_14_moe::routed_experts":3.9145550441458165e-06,"param-mean/_layer_34_gated_delta::out_proj":5.876244176761247e-05,"update-absmax/_layer_10_gated_delta::out_norm":0.00390625,"hidden-states-rms/_layer_01_residual_mlp":0.37958244378664874,"hidden-states-l1/_layer_41_mlp":0.28729814735628934,"hidden-states-l1/_layer_31_gated_delta::beta":1.5734385935590975,"grad-rms/layer_42/gated_delta::conv1d":1.8973907626600528e-05,"hidden-states-absmax/_layer_09_gated_delta::beta":14.5625,"update-rms/_layer_02_gated_delta::A_log":0.0007258080281915047,"update-absmax/_layer_33_gated_delta::out_proj":0.001953125,"hidden-states-rms/_layer_36_moe::shared_experts":0.2562565676306739,"update-mean/_layer_20_gated_delta::dt_bias":4.4607833842746913e-05,"hidden-states-mean/_layer_35_mlp":-0.004373928122094646,"grad-absmax/layer_52/moe::shared_experts":0.00011000368249369785,"hidden-states-l1/_layer_09_gated_delta::core_attention":0.0011166876814663626,"param-mean/_layer_06_gated_delta::conv1d":-0.000646187923848629,"hidden-states-l1/_layer_45_gated_delta::gate":1.5825330713430728,"param-mean/_layer_55_gated_delta::dt_bias":0.8285908699035645,"hidden-states-absmax/_layer_32_moe::router":0.9999680519104004,"hidden-states-mean/_layer_16_gated_delta::out_norm":-0.0003888405508097173,"param-rms/_layer_11_gated_delta::out_proj":0.04475094316047384,"update-mean/_layer_53_gated_delta::out_proj":-1.704800212110058e-07,"grad-absmax/layer_10/gated_delta::A_log":1.1671166248561349e-05,"param-mean/_layer_33_gated_delta::conv1d":-0.0004897498874925077,"hidden-states-mean/_layer_40_input_layernorm":0.0045567532845441445,"param-absmax/_layer_10_gated_delta::out_norm":0.89453125,"hidden-states-absmax/_layer_54_pre_mlp_layernorm":8.5625,"hidden-states-l1/_layer_20_moe::shared_experts":0.09742630689457737,"hidden-states-mean/_layer_35_gated_delta::g":-7.241045861368878,"hidden-states-l1/_layer_01_mlp":0.13816984154818554,"hidden-states-mean/_layer_28_gated_delta::core_attention":0.00017342188833095292,"hidden-states-absmax/_layer_02_residual_mlp":19.875,"update-l1/_layer_01_mlp::linear_fc1":0.00014857515634503216,"hidden-states-l1/_layer_34_pre_mlp_layernorm":0.14272749635308937,"hidden-states-mean/_layer_38_gated_delta::core_attention":-8.833763361380108e-07,"update-mean/_layer_42_gated_delta::conv1d":2.018950181081891e-05,"grad-rms/layer_28/gated_delta::out_norm":1.4352433528115497e-05,"hidden-states-rms/_layer_30_gated_delta::core_attention":0.0013521497647570936,"hidden-states-l1/_layer_37_residual_attn":2.2827048638707237,"update-mean/_layer_29_gated_delta::conv1d":-1.9717117538675666e-06,"param-mean/_layer_25_gated_delta::out_proj":-1.527719541627448e-05,"hidden-states-mean/_layer_08_gated_delta::query":0.012356642244057922,"hidden-states-rms/_layer_18_gated_delta::out_norm":0.08384423373460333,"hidden-states-l1/_layer_29_gated_delta::g":8.57834059933908,"grad-absmax/layer_44/moe::router":0.00010380866297055036,"hidden-states-mean/_layer_11_moe::router":0.010233603802697468,"update-l1/_layer_47_attention::linear_qkv":0.0001340036396868527,"hidden-states-absmax/_layer_23_moe::router":0.9941164255142212,"grad-rms/layer_19/moe::shared_expert_fc2":2.871366493578949e-07,"hidden-states-absmax/_layer_05_gated_delta::query":3.328125,"hidden-states-absmax/_layer_07_gated_delta::out_norm":16.5,"hidden-states-l1/_layer_01_gated_delta::out_norm":0.019940494142829675,"grad-absmax/layer_44/gated_delta::out_proj":9.605105151422322e-05,"update-absmax/_layer_13_attention::linear_qkv":0.001953125,"hidden-states-rms/_layer_38_gated_delta::in_proj":1.2438480987061704,"hidden-states-l1/_layer_09_gated_delta::beta":1.6648161011715057,"hidden-states-absmax/_layer_34_gated_delta::gate":24.625,"hidden-states-absmax/_layer_23_gated_delta::gate":11.4375,"hidden-states-l1/_layer_47_attention::q_layernorm":1.1668964823361019,"hidden-states-mean/_layer_02_residual_mlp":0.0009859275316807666,"hidden-states-absmax/_layer_56_mlp":564,"hidden-states-l1/_layer_19_moe::router":0.01026176203477574,"update-rms/_layer_33_gated_delta::out_proj":0.00021022660454192378,"param-l1/_layer_10_gated_delta::out_proj":0.03516772389411926,"hidden-states-mean/_layer_52_gated_delta::gate":-0.08074102023136857,"hidden-states-rms/_layer_51_residual_attn":11.674656018582994,"hidden-states-l1/_layer_46_gated_delta::out_norm":0.23661425550153917,"hidden-states-rms/_layer_27_gated_delta::gate":1.1046144248270697,"update-rms/_layer_44_gated_delta::out_norm":0.001953125,"hidden-states-absmax/pre_mlp_layernorm":15.3125,"hidden-states-absmax/_layer_10_gated_delta::beta":10.5625,"update-rms/_layer_28_gated_delta::conv1d":0.0007635483323775038,"hidden-states-absmax/_layer_36_gated_delta::in_proj":58.5,"moe/layer_35/act_rms_median":0.13174185156822205,"hidden-states-absmax/_layer_11_moe::routed_experts":10.1875,"update-l1/_layer_02_gated_delta::conv1d":0.000642799714114517,"update-rms/_layer_45_gated_delta::conv1d":0.0008048214579204325,"hidden-states-mean/_layer_09_gated_delta::core_attention":8.808550368989302e-05,"hidden-states-mean/_layer_46_gated_delta::value":0.014980812480248509,"grad-absmax/layer_14/moe::shared_experts":6.05163177169743e-06,"param-absmax/_layer_17_gated_delta::out_norm":1.09375,"hidden-states-rms/_layer_55_gated_delta::out_norm":0.9316756119478854,"hidden-states-rms/_layer_55_gated_delta::value":0.11660858345814099,"grad-absmax/layer_15/moe::shared_experts":3.8714706533937715e-06,"update-l1/_layer_34_gated_delta::dt_bias":0.000524864939507097,"hidden-states-mean/_layer_11_residual_attn":0.0002585305209196235,"update-rms/embedding":0.0017723883868589238,"hidden-states-mean/_layer_20_input_layernorm":0.004159123666927059,"hidden-states-l1/_layer_34_gated_delta::query":0.12944161944885485,"hidden-states-absmax/_layer_34_gated_delta::core_attention":0.375,"param-absmax/_layer_56_gated_delta::out_proj":3.34375,"hidden-states-l1/_layer_15_moe::shared_experts":0.05410765011401774,"grad-rms/layer_22/moe::shared_expert_fc1":3.8893550622638e-07,"hidden-states-rms/_layer_25_moe::shared_experts":0.16755738895247815,"param-rms/_layer_47_attention::o_proj":0.054508390632566205,"hidden-states-absmax/_layer_28_gated_delta::out_norm":60,"update-mean/_layer_42_gated_delta::A_log":-9.414931992068887e-05,"hidden-states-absmax/_layer_20_gated_delta::out_norm":17.625,"hidden-states-rms/_layer_19_gated_delta::g":7.146303311220097,"hidden-states-rms/_layer_40_gated_delta::out_proj":0.5399332937268138,"update-rms/_layer_32_gated_delta::A_log":0.0007699727181551083,"hidden-states-mean/_layer_34_gated_delta::in_proj":-0.250944302494784,"hidden-states-l1/_layer_14_mlp":0.09003180593631338,"hidden-states-mean/_layer_53_moe::routed_experts":-0.00031725895703798104,"grad-absmax/layer_47/moe::expert_fc2":4.66756409878144e-06,"hidden-states-mean/_layer_37_mlp":-0.0016447580051878697,"update-l1/_layer_44_gated_delta::dt_bias":0.0005200258456170559,"hidden-states-absmax/_layer_54_gated_delta::query":6.15625,"hidden-states-absmax/_layer_23_gated_delta::in_proj":17.25,"hidden-states-mean/_layer_15_moe::shared_experts":3.837936404166612e-05,"grad-rms/layer_10/moe::expert_fc1":1.64279272653598e-08,"hidden-states-absmax/_layer_11_gated_delta::value":4.125,"hidden-states-rms/_layer_27_gated_delta::core_attention":0.0020803417890712197,"grad-absmax/layer_28/gated_delta::A_log":1.587492079124786e-05,"update-mean/_layer_11_gated_delta::conv1d":4.146316769038094e-06,"grad-absmax/layer_25/moe::router":6.872083758935332e-05,"hidden-states-mean/_layer_09_gated_delta::beta":-0.7398391651915509,"update-rms/_layer_18_gated_delta::out_proj":0.00020290094336704062,"grad-absmax/layer_04/moe::router":7.6280914072413e-05,"hidden-states-mean/_layer_01_gated_delta::core_attention":0.00697014750184387,"hidden-states-l1/_layer_32_gated_delta::gate":1.3459767515631618,"grad-absmax/layer_42/gated_delta::dt_bias":0.0002394911425653845,"update-absmax/_layer_56_gated_delta::out_norm":0.0078125,"param-mean/_layer_17_gated_delta::conv1d":-0.00012633104051928967,"grad-absmax/layer_17/gated_delta::A_log":1.1382879165466875e-05,"hidden-states-absmax/_layer_41_input_layernorm":31.625,"grad-rms/layer_29/gated_delta::out_proj":4.253145757586591e-07,"hidden-states-mean/_layer_35_gated_delta::in_proj":-0.1630663377800777,"hidden-states-mean/_layer_26_gated_delta::in_proj":-0.1175720184299816,"hidden-states-mean/_layer_43_gated_delta::core_attention":1.627833456371615e-05,"hidden-states-rms/_layer_32_moe::shared_experts":0.19116657363364728,"hidden-states-l1/_layer_25_gated_delta::g":7.235387710975136,"param-mean/_layer_31_gated_delta::A_log":1.2498875856399536,"update-mean/_layer_54_gated_delta::conv1d":1.6514954040758312e-05,"grad-rms/layer_43/moe::shared_expert_fc1":2.845359364469534e-07,"update-l1/_layer_49_attention::linear_qkv":0.0001345457130810246,"grad-rms/layer_48/moe::expert_fc1":1.9183265713946286e-08,"param-rms/_layer_28_gated_delta::out_norm":0.8398001016319233,"update-absmax/_layer_12_gated_delta::out_norm":0.00390625,"hidden-states-rms/_layer_38_gated_delta::key":0.10113094096919976,"update-rms/_layer_06_gated_delta::conv1d":0.0007968678332602315,"hidden-states-absmax/_layer_39_gated_delta::out_proj":61.25,"hidden-states-mean/_layer_31_gated_delta::in_proj":-0.1896336934250908,"update-rms/_layer_01_gated_delta::conv1d":0.0007874717356237218,"grad-rms/layer_31/moe::expert_fc1":1.9250856434495842e-08,"hidden-states-mean/_layer_36_residual_attn":-0.06127721801724648,"param-absmax/_layer_36_gated_delta::A_log":2.2440638542175293,"hidden-states-l1/_layer_08_gated_delta::in_proj":0.6640601607014371,"update-rms/_layer_12_gated_delta::dt_bias":0.0005268343925555399,"hidden-states-rms/_layer_06_moe::routed_experts":0.027442166145955158,"grad-rms/layer_32/moe::shared_experts":1.6988780166560603e-06,"update-mean/_layer_05_gated_delta::dt_bias":-8.178706048056483e-05,"param-l1/_layer_38_gated_delta::A_log":0.9921497106552124,"hidden-states-absmax/_layer_55_gated_delta::beta":14.75,"grad-rms/layer_39/moe::shared_expert_fc1":2.1164401460110552e-07,"hidden-states-mean/_layer_38_gated_delta::beta":0.7045704966271515,"update-l1/_layer_53_gated_delta::out_proj":0.00014236803690437227,"moe/layer_40/act_rms_median":0.17048752307891846,"hidden-states-l1/_layer_38_residual_mlp":2.3894474970635766,"hidden-states-absmax/_layer_38_gated_delta::value":9.9375,"hidden-states-mean/_layer_04_gated_delta::g":-3.663626630738572,"hidden-states-absmax/_layer_55_gated_delta::query":5.75,"hidden-states-l1/_layer_12_input_layernorm":0.1851128858462436,"grad-rms/layer_07/moe::shared_expert_fc2":1.3308752390579564e-07,"hidden-states-absmax/_layer_29_gated_delta::gate":20.625,"param-rms/_layer_43_gated_delta::out_proj":0.043604462721172806,"grad-rms/layer_20/gated_delta::A_log":4.5123577329849415e-06,"hidden-states-absmax/_layer_55_gated_delta::out_proj":250,"grad-absmax/layer_43/gated_delta::dt_bias":1.15477159852162e-05,"hidden-states-mean/_layer_14_residual_mlp":-0.0007812267634989933,"grad-absmax/mlp::linear_fc1":9.269818110624328e-05,"grad-absmax/layer_54/moe::expert_fc2":3.211837338312762e-06,"hidden-states-l1/_layer_38_gated_delta::in_proj":0.8220227862720995,"update-mean/_layer_08_gated_delta::dt_bias":-0.00032550166361033916,"moe/layer_43/act_rms_max":0.48824650049209595,"grad-rms/layer_11/moe::expert_fc2":2.332452778350599e-08,"grad-absmax/layer_02/gated_delta::out_proj":1.9227258235332556e-05,"update-l1/_layer_06_gated_delta::dt_bias":0.00038686906918883324,"hidden-states-l1/_layer_48_residual_attn":3.850791198109531,"hidden-states-rms/_layer_40_gated_delta::beta":2.6180033958722526,"param-rms/_layer_54_gated_delta::dt_bias":0.6597414688738313,"update-mean/_layer_08_gated_delta::out_proj":-3.652167990253474e-08,"param-l1/_layer_41_gated_delta::out_proj":0.03397402539849281,"grad-absmax/layer_41/gated_delta::conv1d":0.0019291372736915946,"hidden-states-absmax/_layer_26_gated_delta::beta":6.90625,"hidden-states-l1/_layer_53_moe::shared_experts":0.5690034114126458,"hidden-states-rms/_layer_02_gated_delta::gate":0.9070979671486509,"param-l1/_layer_51_gated_delta::out_norm":0.993896484375,"param-rms/_layer_37_gated_delta::out_proj":0.04502149606583505,"hidden-states-mean/_layer_14_gated_delta::gate":-0.34441466264899523,"grad-rms/layer_47/moe::expert_fc2":2.085638455362805e-08,"param-mean/_layer_20_gated_delta::conv1d":9.21803803066723e-05,"hidden-states-absmax/_layer_41_gated_delta::key":23.625,"hidden-states-absmax/_layer_42_moe::shared_experts":25.5,"grad-rms/layer_35/moe::shared_experts":2.101291695970399e-06,"hidden-states-l1/_layer_11_gated_delta::core_attention":0.0010610324198493267,"hidden-states-absmax/_layer_31_gated_delta::gate":19,"grad-rms/layer_09/moe::expert_fc2":1.985869055542452e-08,"hidden-states-l1/_layer_29_moe::shared_experts":0.2595305752194613,"update-l1/_layer_37_gated_delta::A_log":0.0004949406138621271,"hidden-states-absmax/_layer_50_pre_mlp_layernorm":15.3125,"grad-rms/layer_25/gated_delta::out_proj":4.1145913381081756e-07,"hidden-states-mean/_layer_44_residual_mlp":-0.09605145109313143,"update-mean/_layer_31_gated_delta::A_log":-0.00012176716700196266,"hidden-states-l1/residual_mlp":2.0302734791600594,"param-absmax/_layer_38_gated_delta::dt_bias":0.7677077651023865,"hidden-states-rms/_layer_20_gated_delta::out_norm":0.15361023182731492,"param-mean/_layer_08_gated_delta::A_log":1.118277668952942,"grad-rms/layer_50/attention::linear_qkv":8.423658998987054e-07,"update-l1/_layer_04_gated_delta::out_norm":0.0005645751953125,"hidden-states-l1/_layer_40_moe::router":0.010084047653722391,"hidden-states-absmax/_layer_16_gated_delta::out_norm":15.375,"hidden-states-l1/_layer_15_moe::router":0.01024868253845891,"hidden-states-l1/_layer_56_gated_delta::in_proj":2.4217162352871586,"grad-absmax/layer_48/moe::shared_expert_fc2":4.5218297600513324e-05,"moe/layer_21/act_rms_min":0.03650236874818802,"hidden-states-l1/_layer_13_moe::router":0.0102446517737153,"update-l1/_layer_13_attention::linear_qkv":0.00014014530461281538,"param-rms/_layer_46_gated_delta::out_proj":0.04158273111742576,"param-rms/_layer_17_gated_delta::dt_bias":0.19676958168187716,"hidden-states-rms/_layer_46_gated_delta::g":6.071996161951121,"hidden-states-mean/_layer_54_pre_mlp_layernorm":-0.00046949885794506763,"moe/layer_45/act_rms_median":0.2055455446243286,"grad-absmax/layer_07/gated_delta::dt_bias":4.151471785007743e-06,"hidden-states-mean/_layer_32_moe::shared_experts":-0.0010329705031777029,"hidden-states-l1/_layer_24_mlp":0.26811591006707075,"hidden-states-absmax/_layer_07_input_layernorm":11.6875,"moe/layer_39/act_rms_max":0.30384212732315063,"hidden-states-rms/_layer_51_mlp":1.5994707751584942,"hidden-states-l1/_layer_39_gated_delta::query":0.07117072596201014,"grad-absmax/layer_45/moe::shared_expert_fc2":5.839141522301361e-05,"param-absmax/_layer_11_gated_delta::dt_bias":0.4767530858516693,"hidden-states-rms/_layer_24_pre_mlp_layernorm":0.16952842806384064,"hidden-states-mean/_layer_41_input_layernorm":0.006578200815254547,"moe/layer_03/act_rms_max":0.03979819640517235,"hidden-states-rms/_layer_43_moe::router":0.07216878508256834,"hidden-states-mean/_layer_12_mlp":-0.0006464481309478737,"hidden-states-absmax/_layer_07_gated_delta::key":3.03125,"update-l1/_layer_13_attention::o_proj":0.00011696853471221402,"hidden-states-rms/_layer_06_mlp":0.0729850048154068,"hidden-states-rms/_layer_44_gated_delta::out_proj":0.8593880210963504,"hidden-states-mean/_layer_15_moe::router":0.01024868253845891,"param-l1/_layer_01_gated_delta::out_proj":0.030059723183512688,"hidden-states-l1/_layer_42_gated_delta::query":0.05030155036467914,"update-l1/_layer_43_gated_delta::A_log":0.000495760643389076,"hidden-states-absmax/_layer_30_residual_attn":502,"update-rms/_layer_50_attention::linear_qkv":0.0001926708410401409,"hidden-states-rms/_layer_38_gated_delta::gate":1.2564960285689393,"param-absmax/_layer_48_gated_delta::dt_bias":1.1082943677902222,"update-mean/_layer_28_gated_delta::conv1d":2.1690891571779503e-06,"grad-absmax/layer_45/moe::shared_experts":2.716365452215541e-05,"grad-absmax/layer_55/moe::shared_expert_fc1":1.6105344911920838e-05,"param-absmax/_layer_06_gated_delta::conv1d":0.498046875,"param-absmax/_layer_29_gated_delta::out_proj":0.9375,"hidden-states-absmax/_layer_43_gated_delta::beta":8.75,"hidden-states-rms/_layer_49_mlp":0.861824964459755,"hidden-states-rms/_layer_40_residual_mlp":7.746239488179858,"param-l1/_layer_54_gated_delta::out_norm":1.4852294921875,"grad-absmax/layer_32/moe::shared_expert_fc1":2.9133903808542527e-05,"hidden-states-mean/_layer_52_moe::routed_experts":-0.00014254600470231272,"hidden-states-absmax/_layer_48_input_layernorm":25.875,"hidden-states-mean/_layer_51_mlp":0.006410989277629766,"grad-rms/embedding":1.673261527961476e-08,"param-l1/_layer_39_gated_delta::conv1d":0.03275974467396736,"update-absmax/_layer_21_attention::linear_qkv":0.00390625,"param-l1/_layer_08_gated_delta::conv1d":0.03490482270717621,"param-mean/_layer_40_gated_delta::out_norm":0.722320556640625,"hidden-states-absmax/_layer_45_residual_mlp":502,"hidden-states-rms/_layer_55_gated_delta::core_attention":0.00716665777369546,"hidden-states-l1/_layer_32_mlp":0.21811164535422867,"grad-absmax/layer_49/attention::o_proj":4.2111510992981493e-05,"grad-rms/layer_23/moe::shared_expert_fc2":5.044129869736294e-07,"param-rms/_layer_16_gated_delta::conv1d":0.0715158884230303,"param-absmax/_layer_15_gated_delta::out_proj":1.0546875,"hidden-states-absmax/_layer_38_residual_mlp":502,"hidden-states-mean/_layer_14_gated_delta::key":0.017203772628757073,"grad-absmax/layer_18/gated_delta::dt_bias":1.32429941004375e-05,"grad-absmax/layer_10/moe::shared_expert_fc1":9.282775863539428e-06,"grad-absmax/layer_33/moe::shared_experts":9.98004361463245e-06,"hidden-states-rms/_layer_51_gated_delta::query":1.0509053042560463,"grad-absmax/layer_51/gated_delta::conv1d":0.011492892168462276,"hidden-states-rms/_layer_20_gated_delta::out_proj":0.3176515892430936,"hidden-states-rms/_layer_43_residual_mlp":8.107705370587178,"hidden-states-rms/_layer_21_moe::shared_experts":0.15365487249858203,"hidden-states-l1/_layer_44_pre_mlp_layernorm":0.1861127240792184,"hidden-states-absmax/_layer_03_gated_delta::beta":10.6875,"grad-absmax/layer_41/moe::expert_fc2":2.0288998712203465e-06,"hidden-states-rms/_layer_39_input_layernorm":0.4919319387919351,"grad-absmax/layer_12/gated_delta::out_norm":0.00020915661298204213,"grad-absmax/layer_40/moe::shared_expert_fc1":9.194377526000608e-06,"hidden-states-absmax/_layer_14_gated_delta::beta":16.75,"update-mean/_layer_14_gated_delta::A_log":-0.0001187371090054512,"moe/layer_21/act_rms_median":0.08498766273260117,"hidden-states-mean/_layer_36_moe::routed_experts":-0.00025446702926227097,"update-l1/_layer_07_gated_delta::conv1d":0.0006771921180188656,"update-l1/_layer_55_gated_delta::A_log":0.0005550924688577652,"param-rms/_layer_37_gated_delta::A_log":1.1413962487129208,"moe/layer_10/act_rms_min_to_median":0.3682459714534158,"hidden-states-mean/_layer_46_pre_mlp_layernorm":0.0015561894506143869,"grad-rms/layer_43/gated_delta::dt_bias":2.634415971836863e-06,"hidden-states-absmax/_layer_23_gated_delta::key":8.125,"update-mean/_layer_11_gated_delta::out_proj":-6.038234801053477e-09,"grad-rms/layer_28/gated_delta::out_proj":5.214229114605825e-07,"hidden-states-absmax/_layer_01_residual_attn":3.15625,"hidden-states-rms/_layer_37_residual_mlp":7.274944578897117,"update-rms/_layer_06_gated_delta::out_proj":0.00021444370365536947,"hidden-states-absmax/_layer_44_gated_delta::in_proj":30.625,"hidden-states-rms/_layer_38_moe::routed_experts":0.15689134574319985,"hidden-states-rms/_layer_44_gated_delta::out_norm":0.4271126231846042,"hidden-states-mean/_layer_30_moe::routed_experts":-0.00016775679673741421,"hidden-states-l1/_layer_11_moe::routed_experts":0.024680249466736505,"grad-rms/layer_35/moe::expert_fc2":2.2337718383475755e-08,"moe/layer_15/act_rms_median":0.04618147015571594,"grad-rms/gated_delta::out_norm":2.4708049022136415e-05,"hidden-states-rms/_layer_39_residual_mlp":7.527981192540267,"moe/layer_41/act_rms_min_to_median":0.4871730846325169,"hidden-states-rms/_layer_44_pre_mlp_layernorm":0.2613058119528444,"grad-absmax/layer_18/moe::shared_experts":7.987721801328007e-06,"hidden-states-absmax/_layer_25_input_layernorm":19.25,"hidden-states-rms/_layer_15_gated_delta::out_norm":0.0624435400687504,"update-mean/_layer_18_gated_delta::out_norm":-0.000274658203125,"grad-absmax/layer_18/moe::shared_expert_fc1":1.5191359125310555e-05,"grad-absmax/layer_56/moe::shared_experts":3.129444667138159e-05,"hidden-states-rms/_layer_18_pre_mlp_layernorm":0.12215496908100175,"update-mean/_layer_04_gated_delta::A_log":-2.166139893233776e-06,"hidden-states-mean/_layer_12_pre_mlp_layernorm":0.0002873321988419829,"hidden-states-absmax/_layer_12_mlp":53.75,"hidden-states-rms/_layer_46_residual_mlp":9.31646904304245,"hidden-states-absmax/_layer_24_moe::shared_experts":31.625,"grad-rms/layer_47/attention::linear_qkv":1.0410099638056498e-06,"grad-absmax/layer_02/mlp::linear_fc1":9.269818110624328e-05,"param-mean/attention::o_proj":-1.3258251751072489e-05,"hidden-states-l1/_layer_43_pre_mlp_layernorm":0.1669731041160946,"hidden-states-l1/_layer_21_moe::routed_experts":0.04220118296488811,"param-rms/_layer_36_gated_delta::out_norm":1.0946054655368322,"moe/layer_02/act_rms_max":0.040762174874544144,"grad-rms/layer_52/moe::shared_experts":5.971543903770803e-06,"param-mean/_layer_22_attention::linear_qkv":0.00020530866459012032,"hidden-states-l1/_layer_12_mlp":0.10004227310923017,"update-absmax/_layer_38_gated_delta::A_log":0.0013331770896911621,"update-rms/_layer_46_gated_delta::conv1d":0.0007263880893925357,"hidden-states-rms/_layer_54_gated_delta::key":0.13083716172365786,"hidden-states-absmax/_layer_35_mlp":34.25,"grad-absmax/layer_55/gated_delta::dt_bias":7.14891793904826e-05,"hidden-states-absmax/_layer_41_gated_delta::out_proj":30,"hidden-states-l1/_layer_20_input_layernorm":0.15365094288002765,"moe/layer_28/act_rms_median":0.1064525693655014,"hidden-states-rms/_layer_43_input_layernorm":0.5919758089126639,"hidden-states-mean/_layer_50_moe::shared_experts":-0.005546920197089934,"param-l1/mlp::linear_fc1":0.035763779655098915,"grad-absmax/layer_16/moe::shared_expert_fc1":1.2800877811969258e-05,"update-rms/_layer_22_attention::o_proj":0.00018247952020070178,"hidden-states-absmax/_layer_26_gated_delta::out_proj":34.25,"grad-norm":0.24894532561302185,"hidden-states-mean/_layer_56_gated_delta::value":-0.02041384634838657,"hidden-states-rms/_layer_13_attention::q_layernorm":1.663350513237557,"grad-absmax/layer_02/gated_delta::conv1d":0.0001717322156764567,"hidden-states-absmax/_layer_32_gated_delta::key":15.125,"param-rms/_layer_34_gated_delta::out_proj":0.04371748119110598,"moe/layer_22/act_rms_min_to_median":0.2975861520874831,"param-absmax/_layer_04_gated_delta::A_log":2.0285754203796387,"hidden-states-absmax/_layer_03_gated_delta::core_attention":0.1591796875,"grad-absmax/layer_23/gated_delta::A_log":2.101440077240113e-05,"hidden-states-absmax/_layer_21_residual_attn":498,"hidden-states-l1/_layer_33_moe::shared_experts":0.11522577586481283,"hidden-states-absmax/_layer_51_moe::shared_experts":216,"update-mean/_layer_40_gated_delta::conv1d":-1.649287878535688e-05,"update-l1/_layer_01_mlp::linear_fc2":0.00015539828746113926,"hidden-states-mean/_layer_42_gated_delta::g":-3.9875593170594357,"grad-rms/layer_52/moe::expert_fc2":1.6411486337309546e-08,"param-absmax/_layer_35_gated_delta::out_proj":0.78515625,"hidden-states-mean/_layer_55_gated_delta::gate":-1.124684944711321,"hidden-states-l1/_layer_20_gated_delta::out_norm":0.08174286267076286,"hidden-states-l1/_layer_44_gated_delta::beta":1.6206949497097671,"param-l1/_layer_35_gated_delta::A_log":1.137868046760559,"hidden-states-absmax/_layer_20_gated_delta::key":10.0625,"hidden-states-mean/input_layernorm":0.0030218218880085092,"hidden-states-absmax/_layer_24_attention::k_layernorm":28.5,"hidden-states-mean/_layer_32_gated_delta::key":-0.03683582759926648,"param-mean/_layer_01_gated_delta::out_proj":8.189978871087078e-06,"grad-absmax/layer_46/moe::expert_fc1":2.6961918138113106e-06,"moe/layer_04/act_rms_median":0.019747860729694366,"hidden-states-rms/_layer_08_mlp":0.0815545310962693,"hidden-states-mean/_layer_08_gated_delta::gate":-0.18517304184115432,"grad-absmax/layer_13/moe::shared_expert_fc1":3.0498298656311817e-05,"param-rms/_layer_26_gated_delta::A_log":1.0984047072224283,"hidden-states-l1/_layer_50_residual_mlp":4.4285064254410775,"moe/layer_28/act_rms_min_to_median":0.3774409768967794,"hidden-states-mean/_layer_27_mlp":-0.0006163160003614128,"hidden-states-rms/_layer_26_input_layernorm":0.3277249590467766,"param-mean/_layer_11_gated_delta::out_norm":0.29107666015625,"update-absmax/_layer_43_gated_delta::out_proj":0.00390625,"grad-rms/layer_44/moe::expert_fc1":2.0068754817347434e-08,"grad-rms/layer_23/gated_delta::conv1d":2.1623678073762266e-05,"update-rms/_layer_11_gated_delta::conv1d":0.0008272797635784087,"hidden-states-absmax/_layer_50_attention::q_layernorm":19.125,"hidden-states-l1/_layer_17_gated_delta::in_proj":0.7331047277666942,"update-l1/_layer_16_gated_delta::out_norm":0.0006103515625,"grad-rms/layer_29/gated_delta::A_log":1.441312481546951e-06,"grad-rms/layer_29/gated_delta::out_norm":2.2259818739047775e-05,"param-l1/_layer_29_gated_delta::out_norm":0.3919525146484375,"grad-rms/layer_26/gated_delta::A_log":2.07739657792098e-06,"grad-rms/layer_45/gated_delta::A_log":1.927284039409809e-05,"hidden-states-absmax/_layer_46_input_layernorm":61.5,"update-mean/_layer_33_gated_delta::dt_bias":-0.00022773162345401943,"hidden-states-rms/_layer_32_gated_delta::out_norm":0.18147900326994315,"hidden-states-mean/_layer_10_gated_delta::in_proj":-0.1323964749933384,"update-mean/_layer_38_gated_delta::conv1d":-1.8830896806321107e-05,"param-mean/_layer_19_gated_delta::conv1d":0.0005393403116613626,"param-mean/gated_delta::out_proj":3.6634621897157595e-06,"update-absmax/_layer_28_gated_delta::conv1d":0.0029296875,"hidden-states-mean/_layer_13_attention::core_attention":-0.00041261733538938586,"update-l1/_layer_26_gated_delta::conv1d":0.0006523928022943437,"grad-rms/layer_56/moe::router":6.55464371483541e-07,"hidden-states-rms/_layer_33_gated_delta::value":0.10174073720794977,"param-absmax/_layer_13_attention::o_proj":1.015625,"grad-rms/layer_41/gated_delta::A_log":1.652337467453461e-05,"hidden-states-absmax/_layer_27_moe::router":0.9997544288635254,"param-absmax/_layer_20_gated_delta::out_norm":2.234375,"hidden-states-l1/_layer_41_gated_delta::gate":1.0326224604394985,"update-rms/_layer_07_gated_delta::conv1d":0.0008592655536853626,"grad-rms/layer_55/moe::shared_expert_fc2":6.814775592188852e-08,"hidden-states-mean/_layer_33_residual_attn":-0.042149288016472816,"grad-rms/layer_01/gated_delta::conv1d":9.225420996123188e-06,"hidden-states-absmax/_layer_15_gated_delta::value":2.453125,"hidden-states-absmax/_layer_48_moe::shared_experts":60,"update-absmax/_layer_12_gated_delta::conv1d":0.0029296875,"hidden-states-l1/_layer_40_mlp":0.2494395672606906,"moe/layer_31/act_rms_min":0.046488769352436066,"grad-absmax/layer_38/gated_delta::A_log":2.9538850867538713e-05,"hidden-states-l1/_layer_05_gated_delta::beta":1.1985941169970176,"hidden-states-rms/_layer_06_gated_delta::query":0.32143826082288207,"param-rms/embedding":0.3697657262631521,"param-l1/_layer_02_mlp::linear_fc2":0.036276791244745255,"hidden-states-l1/_layer_39_gated_delta::core_attention":0.0006031968651425082,"update-mean/_layer_55_gated_delta::out_norm":0.0001220703125,"hidden-states-rms/_layer_03_mlp":0.05474791113846531,"hidden-states-rms/_layer_39_gated_delta::in_proj":1.4346512963381561,"grad-rms/layer_12/gated_delta::out_proj":5.987296420735325e-07,"hidden-states-l1/_layer_16_gated_delta::gate":0.7567919202794934,"update-rms/_layer_37_gated_delta::out_norm":0.002237585788552656,"hidden-states-mean/_layer_39_pre_mlp_layernorm":0.0002724088005426586,"update-absmax/_layer_34_gated_delta::out_proj":0.00390625,"hidden-states-rms/_layer_27_mlp":0.2568668311462021,"hidden-states-rms/_layer_12_gated_delta::beta":2.7805842713315863,"grad-absmax/layer_36/moe::shared_expert_fc1":1.8318069123779424e-05,"hidden-states-l1/_layer_30_gated_delta::gate":0.6947513454856092,"update-mean/_layer_43_gated_delta::dt_bias":-3.0410010367631912e-05,"hidden-states-l1/_layer_24_input_layernorm":0.3314326800623751,"hidden-states-absmax/_layer_10_gated_delta::in_proj":32.5,"grad-rms/layer_12/gated_delta::A_log":5.172304024422134e-05,"hidden-states-mean/_layer_45_moe::routed_experts":-0.00027633004337087707,"grad-absmax/layer_34/moe::router":3.7879424780840054e-05,"param-rms/_layer_09_gated_delta::dt_bias":0.28516785062421146,"hidden-states-rms/_layer_09_gated_delta::in_proj":1.336495985552104,"hidden-states-absmax/_layer_16_mlp":29.75,"param-absmax/_layer_34_gated_delta::conv1d":0.484375,"update-rms/_layer_27_gated_delta::dt_bias":0.0006411419600694347,"param-l1/_layer_33_gated_delta::out_norm":0.4266357421875,"grad-absmax/layer_13/attention::linear_qkv":0.001140108099207282,"hidden-states-l1/_layer_40_residual_mlp":2.568212576891491,"param-l1/_layer_29_gated_delta::dt_bias":0.18809688091278076,"hidden-states-rms/_layer_35_gated_delta::in_proj":1.7976808956584105,"hidden-states-l1/_layer_38_gated_delta::out_proj":0.28344387214706807,"hidden-states-l1/_layer_33_gated_delta::core_attention":0.002430264196427369,"param-rms/_layer_08_gated_delta::dt_bias":0.2587654390888964,"hidden-states-rms/_layer_53_input_layernorm":0.657891870165813,"hidden-states-absmax/_layer_24_moe::router":0.9999601244926453,"hidden-states-mean/_layer_25_gated_delta::g":-7.232985405085731,"hidden-states-absmax/_layer_01_pre_mlp_layernorm":2.375,"hidden-states-l1/_layer_31_gated_delta::value":0.04115920631995276,"hidden-states-mean/_layer_27_input_layernorm":0.0006393317778915796,"param-mean/_layer_32_gated_delta::A_log":1.0632824897766113,"param-l1/_layer_37_gated_delta::A_log":0.9958289861679077,"hidden-states-l1/_layer_02_gated_delta::out_proj":0.06837939541094248,"update-rms/_layer_31_gated_delta::dt_bias":0.0007922260499881345,"grad-rms/layer_53/moe::expert_fc2":1.6188480049588498e-08,"grad-rms/layer_36/moe::shared_expert_fc2":1.7236447940180126e-07,"hidden-states-mean/_layer_33_gated_delta::query":0.046448329412738014,"hidden-states-mean/_layer_30_moe::router":0.010222445935598363,"param-rms/_layer_10_gated_delta::out_proj":0.045205362038453205,"hidden-states-l1/_layer_25_mlp":0.202822366693302,"hidden-states-l1/_layer_17_residual_attn":0.5363559170193574,"update-rms/_layer_36_gated_delta::out_proj":0.00018961121902178106,"hidden-states-rms/gated_delta::gate":1.562675033151109,"hidden-states-absmax/_layer_05_residual_attn":43.75,"update-rms/gated_delta::dt_bias":0.0006553415264772941,"update-absmax/_layer_12_gated_delta::A_log":0.0010796189308166504,"hidden-states-l1/_layer_40_input_layernorm":0.32638601379227805,"update-rms/_layer_01_mlp::linear_fc2":0.00021643502008223862,"param-l1/_layer_05_gated_delta::dt_bias":0.346110075712204,"hidden-states-mean/_layer_09_pre_mlp_layernorm":8.90043772105689e-05,"update-mean/_layer_35_gated_delta::conv1d":4.338702183304122e-06,"param-mean/_layer_38_gated_delta::A_log":0.8418731689453125,"update-mean/_layer_36_gated_delta::dt_bias":-0.00015312200412154198,"update-mean/_layer_37_gated_delta::conv1d":-1.3002089644942316e-06,"hidden-states-l1/_layer_21_attention::core_attention":0.24928952745074362,"hidden-states-mean/_layer_53_gated_delta::query":0.005583937014214636,"hidden-states-absmax/_layer_06_gated_delta::in_proj":37.5,"update-absmax/_layer_44_gated_delta::dt_bias":0.0017451643943786621,"grad-absmax/layer_32/moe::expert_fc2":2.026381253017462e-06,"update-absmax/_layer_10_gated_delta::A_log":0.000948488712310791,"grad-rms/layer_10/gated_delta::dt_bias":2.961478093756293e-06,"hidden-states-rms/_layer_37_gated_delta::g":3.3421707554185134,"hidden-states-mean/_layer_34_gated_delta::value":0.007742336942832106,"hidden-states-l1/_layer_53_residual_attn":5.994430302909649,"moe/layer_37/act_rms_median":0.14042940735816956,"grad-absmax/layer_10/gated_delta::out_norm":7.59498798288405e-05,"hidden-states-absmax/_layer_28_gated_delta::gate":11.875,"hidden-states-l1/_layer_09_gated_delta::value":0.038826347300570885,"hidden-states-rms/_layer_36_gated_delta::key":0.058957583179489015,"param-mean/_layer_45_gated_delta::out_norm":0.85382080078125,"param-rms/_layer_20_gated_delta::out_norm":1.0922420460801452,"hidden-states-absmax/_layer_21_mlp":17.375,"update-rms/_layer_53_gated_delta::conv1d":0.0007743628078065947,"hidden-states-l1/_layer_13_mlp":0.09172941347711247,"hidden-states-absmax/_layer_23_moe::shared_experts":28.375,"hidden-states-l1/_layer_52_input_layernorm":0.34098748457282596,"grad-rms/layer_31/moe::shared_experts":1.5740134022943605e-06,"update-mean/_layer_10_gated_delta::dt_bias":-5.981969297863543e-05,"grad-absmax/layer_15/gated_delta::out_proj":1.643122232053429e-05,"param-rms/_layer_20_gated_delta::conv1d":0.08023906445194119,"grad-rms/layer_01/gated_delta::out_proj":1.119755761281535e-06,"grad-rms/layer_35/moe::shared_expert_fc2":1.8402785121028096e-07,"hidden-states-mean/_layer_16_moe::router":0.010256911706036202,"hidden-states-l1/_layer_29_gated_delta::out_norm":0.06822639026956442,"param-l1/_layer_16_gated_delta::out_norm":0.5567626953125,"grad-absmax/layer_32/gated_delta::A_log":2.1499681679415517e-06,"hidden-states-l1/_layer_00_embeddings":0.21960930424968036,"update-l1/_layer_30_gated_delta::dt_bias":0.00043827109038829803,"hidden-states-mean/_layer_10_gated_delta::out_proj":-6.512099983914567e-05,"hidden-states-absmax/_layer_53_residual_attn":2768,"hidden-states-absmax/_layer_52_gated_delta::core_attention":1.046875,"hidden-states-absmax/_layer_37_gated_delta::beta":12.3125,"param-mean/_layer_18_gated_delta::conv1d":7.424796785926446e-05,"hidden-states-mean/_layer_05_mlp":-0.00011265088575362542,"param-l1/_layer_27_gated_delta::A_log":0.913462221622467,"hidden-states-l1/_layer_51_gated_delta::gate":1.9668965300198233,"param-rms/_layer_51_gated_delta::out_norm":1.022011965284085,"hidden-states-rms/_layer_24_input_layernorm":0.68710395694714,"hidden-states-rms/_layer_25_gated_delta::beta":2.035930351455926,"hidden-states-absmax/input_layernorm":61.5,"update-l1/_layer_17_gated_delta::out_norm":0.0009002685546875,"param-mean/_layer_01_mlp::linear_fc1":2.5679633836261928e-05,"hidden-states-absmax/_layer_29_residual_mlp":502,"hidden-states-mean/_layer_07_gated_delta::value":-0.0016155069058620222,"hidden-states-mean/_layer_25_gated_delta::query":0.025384142295615947,"param-l1/_layer_32_gated_delta::dt_bias":0.22803524136543274,"update-l1/_layer_53_gated_delta::A_log":0.000494158361107111,"hidden-states-rms/_layer_07_gated_delta::gate":1.1423919424348272,"hidden-states-rms/attention::core_attention":0.5058701716156355,"hidden-states-l1/_layer_18_input_layernorm":0.13987545846408111,"param-l1/gated_delta::dt_bias":0.34783059784344267,"hidden-states-l1/_layer_14_moe::shared_experts":0.05501789137934344,"param-absmax/_layer_55_gated_delta::A_log":2.8615500926971436,"hidden-states-l1/_layer_25_gated_delta::beta":1.6572253295451795,"hidden-states-mean/_layer_29_gated_delta::gate":-0.24377052411681702,"grad-absmax/layer_51/moe::expert_fc2":1.8052931409329176e-05,"grad-absmax/layer_37/moe::shared_experts":5.8794971664610785e-06,"hidden-states-rms/_layer_15_gated_delta::core_attention":0.0014916317797252403,"hidden-states-l1/_layer_17_moe::router":0.010252138987523553,"hidden-states-l1/_layer_54_moe::router":0.010351421715048426,"hidden-states-absmax/_layer_30_moe::routed_experts":14,"hidden-states-mean/_layer_35_input_layernorm":0.0037790406343687338,"param-absmax/_layer_32_gated_delta::out_proj":1.59375,"hidden-states-absmax/_layer_46_gated_delta::gate":30.25,"param-rms/_layer_27_gated_delta::out_norm":0.7899182277923857,"param-absmax/_layer_25_gated_delta::conv1d":0.76171875,"hidden-states-rms/_layer_32_gated_delta::out_proj":0.4095907098371874,"hidden-states-mean/_layer_03_moe::routed_experts":6.954168414797383e-06,"hidden-states-mean/_layer_30_pre_mlp_layernorm":-1.8505614594294382e-05,"param-rms/_layer_52_gated_delta::dt_bias":0.5978385298267094,"hidden-states-l1/_layer_02_gated_delta::value":0.04593195568704077,"update-absmax/_layer_01_gated_delta::A_log":0.001569509506225586,"update-mean/_layer_16_gated_delta::A_log":-0.00018289987929165363,"grad-rms/layer_40/moe::expert_fc1":1.975483683993685e-08,"hidden-states-rms/_layer_54_gated_delta::beta":2.5229056625207,"param-mean/_layer_29_gated_delta::A_log":0.8318787217140198,"param-l1/_layer_06_gated_delta::dt_bias":0.3868972659111023,"hidden-states-mean/_layer_14_pre_mlp_layernorm":-0.0006045695183658025,"hidden-states-mean/_layer_09_gated_delta::out_proj":-2.0332023873342252e-05,"hidden-states-l1/_layer_02_gated_delta::gate":0.6502185642979817,"update-rms/_layer_55_gated_delta::dt_bias":0.0007081532300093115,"hidden-states-absmax/_layer_09_residual_mlp":65.5,"hidden-states-absmax/_layer_23_gated_delta::core_attention":0.13671875,"hidden-states-mean/_layer_47_residual_mlp":-0.10111717595066738,"hidden-states-absmax/_layer_44_gated_delta::key":5.46875,"param-absmax/_layer_42_gated_delta::conv1d":0.44140625,"hidden-states-mean/_layer_10_mlp":-0.0002749844025336239,"grad-rms/layer_38/gated_delta::out_proj":3.20284123614991e-07,"update-rms/_layer_08_gated_delta::out_norm":0.0011320335077509404,"param-l1/_layer_11_gated_delta::A_log":1.1240043640136719,"hidden-states-absmax/_layer_44_moe::shared_experts":23.125,"param-l1/_layer_26_gated_delta::out_proj":0.03483280912041664,"hidden-states-absmax/_layer_36_gated_delta::beta":9.6875,"hidden-states-absmax/_layer_45_moe::router":0.9968628883361816,"hidden-states-l1/_layer_09_gated_delta::key":0.04611472217616107,"hidden-states-absmax/_layer_15_gated_delta::query":6.21875,"update-absmax/_layer_04_gated_delta::conv1d":0.0034637451171875,"hidden-states-mean/_layer_48_residual_mlp":-0.10345504921138873,"hidden-states-rms/_layer_21_input_layernorm":0.6493002124628325,"update-mean/_layer_37_gated_delta::dt_bias":4.109041765332222e-05,"hidden-states-absmax/_layer_05_pre_mlp_layernorm":2.109375,"hidden-states-l1/_layer_50_attention::q_layernorm":1.08356166882058,"grad-rms/layer_41/moe::shared_experts":2.593561646794653e-06,"update-mean/_layer_07_gated_delta::out_proj":-1.2192630549634487e-07,"hidden-states-mean/_layer_41_moe::shared_experts":-0.0016864237523286048,"hidden-states-mean/_layer_56_gated_delta::key":-0.011676410673726878,"grad-rms/layer_35/gated_delta::dt_bias":2.3376236510135904e-06,"hidden-states-rms/_layer_02_input_layernorm":0.3266204411481782,"hidden-states-mean/_layer_21_residual_attn":-0.0032278750214427736,"param-rms/_layer_54_gated_delta::conv1d":0.05300788680502196,"hidden-states-mean/_layer_02_gated_delta::query":0.01360302766895624,"param-absmax/_layer_01_gated_delta::A_log":2.4765207767486572,"param-rms/_layer_46_gated_delta::conv1d":0.06505968322574619,"hidden-states-l1/_layer_16_input_layernorm":0.1549050992614149,"hidden-states-rms/_layer_32_gated_delta::core_attention":0.010753802059210688,"hidden-states-absmax/_layer_34_residual_mlp":502,"hidden-states-l1/_layer_10_pre_mlp_layernorm":0.06830478528795067,"hidden-states-rms/_layer_01_gated_delta::beta":2.8777845473675137,"hidden-states-absmax/_layer_35_residual_mlp":502,"grad-rms/layer_30/gated_delta::dt_bias":2.280154780146985e-06,"update-rms/_layer_34_gated_delta::dt_bias":0.0006499754430145104,"hidden-states-rms/_layer_17_pre_mlp_layernorm":0.11663633807738522,"grad-rms/layer_53/gated_delta::conv1d":0.0001990039940606139,"hidden-states-rms/_layer_15_gated_delta::out_proj":0.13228879165794002,"update-l1/_layer_16_gated_delta::A_log":0.0005280866753309965,"param-rms/_layer_31_gated_delta::A_log":1.3361881311009114,"grad-rms/layer_35/gated_delta::out_proj":3.116797172503481e-07,"hidden-states-rms/_layer_16_residual_mlp":1.2967981376246709,"update-rms/_layer_16_gated_delta::out_norm":0.0013702358594142852,"hidden-states-l1/_layer_44_moe::routed_experts":0.12701870063517187,"hidden-states-rms/_layer_36_gated_delta::beta":2.427684434020202,"hidden-states-l1/_layer_36_gated_delta::out_proj":0.36029554728216495,"hidden-states-absmax/_layer_21_moe::shared_experts":14.0625,"grad-absmax/layer_11/moe::expert_fc1":1.4562920114258304e-06,"grad-rms/layer_15/gated_delta::dt_bias":1.7376241253851792e-06,"grad-rms/layer_46/moe::shared_expert_fc2":5.96171183269146e-07,"hidden-states-mean/_layer_54_gated_delta::core_attention":0.000248194410716076,"update-absmax/_layer_09_gated_delta::A_log":0.0015797317028045654,"grad-rms/layer_50/attention::o_proj":3.0415000454033444e-07,"hidden-states-absmax/_layer_40_gated_delta::out_norm":98,"grad-rms/layer_27/gated_delta::out_proj":3.652536032895641e-07,"param-absmax/_layer_22_attention::o_proj":0.458984375,"hidden-states-mean/_layer_06_gated_delta::value":0.0014489651056635506,"update-rms/_layer_25_gated_delta::A_log":0.0006460163791745973,"hidden-states-absmax/_layer_08_moe::routed_experts":10.9375,"update-rms/_layer_33_gated_delta::A_log":0.0007777691676771423,"hidden-states-rms/_layer_43_gated_delta::core_attention":0.0014196215863019799,"moe/layer_09/act_rms_median":0.034030020236968994,"update-absmax/_layer_02_gated_delta::out_norm":0.001953125,"update-absmax/_layer_04_gated_delta::A_log":0.0014181137084960938,"hidden-states-rms/_layer_37_moe::routed_experts":0.1499143565736144,"update-mean/_layer_23_gated_delta::A_log":-0.00011306488886475563,"moe/layer_29/act_rms_median":0.11615362018346786,"hidden-states-absmax/_layer_51_pre_mlp_layernorm":15.25,"hidden-states-l1/_layer_08_gated_delta::query":0.04340421925833455,"hidden-states-mean/_layer_49_attention::core_attention":0.00013360857111578537,"param-mean/_layer_52_gated_delta::out_proj":4.804025957128033e-05,"hidden-states-l1/_layer_15_pre_mlp_layernorm":0.08429764471026,"update-rms/_layer_16_gated_delta::conv1d":0.0008034292940558498,"hidden-states-rms/_layer_12_input_layernorm":0.4144279805645475,"hidden-states-rms/_layer_32_moe::router":0.07216878508256834,"hidden-states-l1/_layer_18_gated_delta::value":0.022477199208998586,"param-absmax/_layer_08_gated_delta::out_proj":1.4140625,"hidden-states-l1/_layer_07_pre_mlp_layernorm":0.055024064186103845,"grad-absmax/layer_36/gated_delta::conv1d":0.007848570123314857,"hidden-states-mean/_layer_36_gated_delta::gate":-0.11968435283712779,"hidden-states-absmax/_layer_11_moe::shared_experts":9,"hidden-states-rms/_layer_35_moe::shared_experts":0.28277231899633826,"hidden-states-rms/_layer_25_gated_delta::g":7.778573280574064,"hidden-states-absmax/_layer_02_input_layernorm":11,"param-l1/_layer_55_gated_delta::conv1d":0.03047036938369274,"hidden-states-l1/_layer_48_gated_delta::gate":1.1265912206877733,"param-rms/_layer_45_gated_delta::conv1d":0.07634483776223254,"param-l1/_layer_03_gated_delta::out_norm":0.13102340698242188,"hidden-states-rms/_layer_05_gated_delta::value":0.056702260990788456,"hidden-states-mean/_layer_31_gated_delta::beta":-0.27907687772236794,"hidden-states-mean/_layer_35_gated_delta::gate":-0.3018809320784726,"hidden-states-mean/_layer_30_gated_delta::gate":0.04860335050265199,"param-rms/_layer_26_gated_delta::out_proj":0.044902649683862136,"hidden-states-absmax/_layer_18_gated_delta::key":8.375,"grad-rms/mlp::linear_fc2":1.8952704952781052e-07,"update-mean/_layer_07_gated_delta::conv1d":6.637807246079319e-07,"hidden-states-absmax/_layer_01_gated_delta::in_proj":70.5,"param-rms/_layer_10_gated_delta::conv1d":0.0618716313397592,"hidden-states-absmax/_layer_04_residual_attn":39,"hidden-states-mean/_layer_44_gated_delta::value":0.014088977876481756,"param-rms/_layer_29_gated_delta::A_log":1.126661610046491,"grad-rms/layer_21/moe::expert_fc2":2.034632252674716e-08,"update-mean/_layer_10_gated_delta::out_proj":5.03167143506289e-08,"update-mean/_layer_02_gated_delta::out_proj":-4.90067542102679e-08,"hidden-states-mean/_layer_34_gated_delta::out_proj":-0.001844392145129102,"hidden-states-mean/_layer_14_gated_delta::out_proj":-0.0008593391836578171,"grad-rms/layer_04/gated_delta::out_norm":2.759052908918791e-05,"grad-absmax/layer_45/gated_delta::dt_bias":0.00010667149035725743,"param-mean/_layer_11_gated_delta::conv1d":0.0009376379894092679,"hidden-states-mean/_layer_56_mlp":0.022412962158079405,"update-absmax/_layer_08_gated_delta::out_norm":0.00390625,"update-absmax/_layer_11_gated_delta::out_proj":0.00390625,"update-rms/_layer_30_gated_delta::dt_bias":0.0005213912613699241,"grad-absmax/gated_delta::conv1d":0.026494406163692474,"grad-rms/layer_16/moe::expert_fc1":2.035711611180558e-08,"grad-absmax/layer_09/gated_delta::conv1d":0.0005530933849513531,"moe/layer_16/act_rms_max":0.09636622667312622,"param-l1/_layer_06_gated_delta::out_norm":0.23595428466796875,"hidden-states-rms/_layer_06_pre_mlp_layernorm":0.07080723070004072,"grad-rms/layer_42/moe::expert_fc2":1.9984020596589857e-08,"hidden-states-rms/_layer_50_moe::shared_experts":1.1952843471380494,"param-absmax/_layer_40_gated_delta::A_log":1.9271665811538696,"hidden-states-absmax/_layer_51_gated_delta::g":20.375,"hidden-states-l1/_layer_17_gated_delta::beta":2.465546425050071,"grad-absmax/layer_21/attention::o_proj":5.940065602771938e-05,"param-mean/_layer_32_gated_delta::out_proj":-8.300926310766954e-06,"hidden-states-mean/_layer_19_gated_delta::g":-6.693737827985028,"hidden-states-mean/_layer_23_input_layernorm":0.002367113999082327,"hidden-states-l1/_layer_38_mlp":0.22277885350928944,"update-mean/_layer_44_gated_delta::out_norm":-0.000244140625,"moe/layer_03/act_rms_min":0.0035605726297944784,"grad-rms/layer_32/moe::shared_expert_fc2":1.4713916061297623e-07,"hidden-states-mean/_layer_39_gated_delta::beta":0.03044091900478828,"update-mean/_layer_27_gated_delta::A_log":-2.682674676179886e-05,"hidden-states-absmax/_layer_42_residual_mlp":502,"grad-rms/layer_40/moe::router":3.7309097689802106e-07,"hidden-states-l1/_layer_02_gated_delta::out_norm":0.017342629115894927,"hidden-states-l1/_layer_18_pre_mlp_layernorm":0.09109203113222493,"hidden-states-l1/_layer_04_gated_delta::key":0.0406545188313149,"hidden-states-l1/_layer_15_gated_delta::core_attention":0.0006970533535097066,"hidden-states-absmax/_layer_32_mlp":28.5,"grad-absmax/layer_56/gated_delta::out_norm":5.4897118388907984e-05,"hidden-states-mean/_layer_42_gated_delta::out_norm":-0.0014832727050957328,"grad-absmax/layer_15/moe::expert_fc2":9.736397714732448e-07,"grad-rms/layer_36/moe::router":3.5058543267362123e-07,"hidden-states-absmax/_layer_36_pre_mlp_layernorm":4.3125,"hidden-states-rms/_layer_36_input_layernorm":0.679278894259498,"hidden-states-mean/_layer_26_gated_delta::out_norm":-0.001046119033268873,"hidden-states-l1/_layer_49_pre_mlp_layernorm":0.20503287971411702,"hidden-states-mean/_layer_56_moe::shared_experts":0.022417344962638017,"grad-rms/layer_33/moe::router":5.573221460526812e-07,"grad-rms/layer_53/moe::shared_expert_fc2":6.543495591613656e-07,"update-absmax/_layer_34_gated_delta::dt_bias":0.0014875233173370361,"hidden-states-absmax/_layer_06_gated_delta::key":2.953125,"grad-rms/layer_21/moe::shared_expert_fc2":3.6007030387554737e-07,"param-absmax/_layer_42_gated_delta::A_log":1.9496691226959229,"update-l1/_layer_55_gated_delta::out_proj":0.00012477322889026254,"hidden-states-rms/_layer_53_moe::shared_experts":1.6902156756271534,"hidden-states-rms/_layer_51_gated_delta::beta":1.9560292075498855,"hidden-states-absmax/_layer_10_gated_delta::gate":20.375,"hidden-states-l1/_layer_12_moe::routed_experts":0.021167662695775073,"hidden-states-mean/_layer_10_moe::routed_experts":-2.8006909373839305e-05,"param-l1/_layer_12_gated_delta::conv1d":0.03359629213809967,"hidden-states-l1/_layer_15_gated_delta::in_proj":0.6651076013171915,"update-rms/_layer_53_gated_delta::out_proj":0.0002045809329154986,"hidden-states-absmax/_layer_23_gated_delta::beta":8.5625,"hidden-states-rms/_layer_01_mlp":0.3056972533808465,"grad-rms/layer_49/moe::shared_expert_fc1":3.213777059558045e-07,"hidden-states-l1/_layer_47_mlp":0.5826938250891954,"hidden-states-mean/_layer_54_gated_delta::query":-0.013882577211568024,"update-l1/_layer_42_gated_delta::out_proj":0.00014679819287266582,"param-mean/_layer_17_gated_delta::out_norm":0.620819091796875,"hidden-states-mean/_layer_26_pre_mlp_layernorm":0.0001592679808487199,"hidden-states-mean/_layer_29_residual_attn":-0.023270453306490576,"param-mean/_layer_43_gated_delta::conv1d":-0.000338887533871457,"hidden-states-l1/_layer_30_gated_delta::core_attention":0.0006731641117157473,"update-absmax/_layer_33_gated_delta::conv1d":0.00390625,"param-rms/_layer_31_gated_delta::dt_bias":0.3176913830138779,"param-rms/_layer_45_gated_delta::out_norm":0.8703737576360902,"grad-rms/layer_18/moe::shared_experts":1.011175309745952e-06,"hidden-states-l1/_layer_38_gated_delta::out_norm":0.06943371974631783,"hidden-states-mean/_layer_07_residual_attn":0.0014023111942185395,"hidden-states-rms/_layer_33_input_layernorm":0.5538444890016667,"param-mean/_layer_24_attention::linear_qkv":9.012890950543806e-05,"hidden-states-rms/_layer_30_gated_delta::in_proj":1.162987548345349,"hidden-states-rms/_layer_26_gated_delta::core_attention":0.0016820572172807767,"grad-absmax/layer_53/moe::shared_experts":6.200118514243513e-05,"hidden-states-rms/_layer_56_gated_delta::value":0.13482773885938834,"hidden-states-rms/_layer_19_gated_delta::out_proj":0.18308598806543844,"hidden-states-absmax/_layer_35_gated_delta::g":21.375,"grad-absmax/layer_21/moe::expert_fc2":1.9098697521258146e-06,"hidden-states-l1/_layer_31_residual_attn":1.7163217921640626,"hidden-states-absmax/_layer_08_gated_delta::key":3.859375,"update-absmax/_layer_28_gated_delta::out_norm":0.00390625,"update-absmax/mlp::linear_fc2":0.00390625,"grad-absmax/layer_19/gated_delta::out_proj":2.2689735487801954e-05,"hidden-states-l1/_layer_24_moe::routed_experts":0.07195494504634269,"hidden-states-mean/_layer_07_gated_delta::beta":-0.7795701956236596,"grad-absmax/layer_43/moe::shared_experts":9.389620572619606e-06,"grad-rms/layer_05/gated_delta::out_norm":4.105461104470231e-05,"hidden-states-absmax/_layer_26_residual_mlp":502,"hidden-states-mean/_layer_21_input_layernorm":0.003830915307958759,"grad-absmax/layer_20/moe::shared_experts":1.879096271295566e-05,"grad-absmax/layer_32/gated_delta::conv1d":7.885476225055754e-05,"param-l1/_layer_38_gated_delta::out_proj":0.03499220684170723,"param-absmax/_layer_32_gated_delta::out_norm":0.828125,"hidden-states-l1/_layer_06_gated_delta::query":0.06644240813760854,"grad-absmax/layer_56/moe::shared_expert_fc2":2.470639992679935e-05,"hidden-states-rms/gated_delta::out_proj":2.6417304318742887,"update-mean/_layer_04_gated_delta::out_proj":-6.124685647534989e-09,"hidden-states-rms/_layer_47_attention::q_layernorm":1.6862380233986425,"grad-absmax/layer_12/moe::shared_expert_fc2":1.3345393199415412e-05,"hidden-states-rms/_layer_50_residual_attn":10.352341389677589,"hidden-states-absmax/_layer_34_gated_delta::beta":10.625,"hidden-states-l1/_layer_51_moe::shared_experts":0.43022080424079945,"grad-rms/layer_34/moe::expert_fc1":2.0138875487307603e-08,"grad-rms/layer_10/gated_delta::conv1d":1.2261901150115498e-05,"hidden-states-rms/_layer_29_gated_delta::gate":1.3970143818985883,"hidden-states-l1/_layer_54_residual_mlp":6.468853767973634,"moe/layer_41/act_rms_max":0.41023537516593933,"update-rms/_layer_28_gated_delta::A_log":0.0005482966704589189,"hidden-states-rms/_layer_42_gated_delta::beta":1.2817925789741704,"hidden-states-mean/_layer_10_gated_delta::g":-5.651274401101398,"hidden-states-rms/_layer_45_moe::router":0.07216878508256834,"hidden-states-mean/_layer_07_moe::routed_experts":8.702441021096926e-06,"hidden-states-mean/_layer_29_gated_delta::in_proj":-0.16241869866818073,"hidden-states-l1/_layer_06_gated_delta::in_proj":0.9267522483874896,"update-mean/_layer_01_gated_delta::conv1d":-7.302616722881794e-06,"param-mean/_layer_39_gated_delta::out_proj":4.360825187177397e-06,"hidden-states-l1/_layer_41_gated_delta::core_attention":0.0003718154773510816,"grad-absmax/layer_55/gated_delta::out_proj":3.983623537351377e-05,"hidden-states-absmax/_layer_04_moe::router":0.9909022450447083,"hidden-states-rms/_layer_41_residual_attn":7.7809161114020915,"hidden-states-absmax/_layer_56_gated_delta::value":5.875,"grad-rms/layer_16/gated_delta::dt_bias":3.876751454681701e-06,"hidden-states-mean/_layer_42_gated_delta::beta":-0.2262073428878001,"hidden-states-absmax/_layer_09_gated_delta::gate":33,"update-mean/_layer_52_gated_delta::out_norm":-0.00042724609375,"hidden-states-l1/_layer_33_residual_mlp":1.933930985204099,"hidden-states-absmax/_layer_26_residual_attn":500,"hidden-states-rms/_layer_51_gated_delta::out_norm":3.032690771514885,"param-l1/_layer_30_gated_delta::dt_bias":0.12995405495166779,"update-absmax/_layer_16_gated_delta::A_log":0.0013117790222167969,"hidden-states-rms/_layer_31_gated_delta::query":0.6229057642994179,"hidden-states-absmax/_layer_46_gated_delta::key":5.21875,"hidden-states-l1/_layer_13_attention::o_proj":0.16067648359256786,"hidden-states-absmax/_layer_55_residual_attn":2800,"hidden-states-rms/_layer_08_input_layernorm":0.26479237322346655,"update-mean/_layer_05_gated_delta::out_proj":1.5386790153115726e-07,"grad-absmax/layer_48/moe::router":3.7150297430343926e-05,"hidden-states-l1/_layer_26_moe::routed_experts":0.05834235202175184,"hidden-states-absmax/_layer_12_moe::routed_experts":49.5,"grad-rms/layer_29/moe::expert_fc2":2.2102647052279425e-08,"grad-absmax/layer_20/gated_delta::out_proj":7.922751683508977e-05,"hidden-states-rms/_layer_03_gated_delta::query":0.08654155067271388,"hidden-states-l1/_layer_29_residual_mlp":1.4971730109659447,"update-mean/_layer_17_gated_delta::dt_bias":0.00014587129408027977,"hidden-states-l1/_layer_05_residual_attn":0.21245178132674472,"hidden-states-absmax/_layer_19_mlp":21.75,"param-rms/_layer_32_gated_delta::out_norm":0.3482118945336596,"hidden-states-rms/mlp":0.7558406617074305,"hidden-states-mean/_layer_17_moe::shared_experts":0.00015670223050553864,"hidden-states-rms/_layer_48_input_layernorm":0.6323141231077845,"hidden-states-mean/_layer_46_gated_delta::beta":-0.18853165966649768,"update-rms/_layer_35_gated_delta::dt_bias":0.0005981885564790269,"grad-rms/layer_15/gated_delta::out_proj":3.923028822242864e-07,"update-absmax/_layer_06_gated_delta::dt_bias":0.0013244450092315674,"update-absmax/_layer_48_gated_delta::dt_bias":0.0014571547508239746,"hidden-states-rms/_layer_29_gated_delta::value":0.07586405826483608,"hidden-states-rms/_layer_18_gated_delta::query":0.1462844461440008,"update-rms/_layer_54_gated_delta::conv1d":0.000716182267580071,"grad-absmax/layer_18/moe::expert_fc2":1.182355276796443e-06,"hidden-states-absmax/_layer_28_gated_delta::core_attention":0.10546875,"grad-absmax/layer_28/gated_delta::out_proj":5.460175088956021e-05,"hidden-states-absmax/_layer_05_gated_delta::in_proj":24.625,"hidden-states-rms/_layer_14_gated_delta::value":0.056116568763175105,"update-mean/_layer_57_lm_head":-1.2675708376264083e-06,"grad-absmax/layer_29/gated_delta::out_proj":4.517806155490689e-05,"hidden-states-l1/_layer_08_gated_delta::beta":0.9731434551648445,"hidden-states-absmax/_layer_29_mlp":113.5,"hidden-states-mean/_layer_11_gated_delta::core_attention":5.434039769398133e-05,"hidden-states-l1/_layer_29_moe::router":0.010247696578136536,"hidden-states-absmax/_layer_27_pre_mlp_layernorm":3.21875,"hidden-states-l1/_layer_22_attention::core_attention":0.235632119850173,"hidden-states-rms/_layer_42_gated_delta::core_attention":0.0007656456675865687,"param-rms/_layer_04_gated_delta::conv1d":0.05512130630670612,"hidden-states-absmax/_layer_38_moe::router":0.9999278783798218,"update-absmax/_layer_26_gated_delta::A_log":0.001379936933517456,"hidden-states-mean/_layer_16_input_layernorm":0.0008831985649181682,"hidden-states-rms/_layer_22_mlp":0.2830486044680088,"hidden-states-mean/_layer_05_gated_delta::value":0.0003701026003091902,"update-absmax/_layer_18_gated_delta::out_proj":0.00390625,"hidden-states-rms/_layer_36_residual_attn":6.514770547311555,"hidden-states-l1/_layer_01_input_layernorm":0.3434783629749254,"hidden-states-l1/_layer_02_mlp":0.07655724305596864,"hidden-states-l1/_layer_20_gated_delta::out_proj":0.21620213582541178,"hidden-states-absmax/_layer_40_moe::routed_experts":22.375,"hidden-states-mean/_layer_13_attention::q_layernorm":-0.13528038683895907,"grad-absmax/layer_06/gated_delta::out_proj":5.247380249784328e-05,"moe/layer_39/act_rms_median":0.15793699026107788,"hidden-states-rms/_layer_28_gated_delta::key":0.05754735257543059,"hidden-states-rms/_layer_25_residual_attn":2.715382285148258,"hidden-states-l1/_layer_08_moe::router":0.01024271089716131,"grad-absmax/layer_20/gated_delta::dt_bias":1.3698604561795946e-05,"hidden-states-absmax/_layer_09_moe::routed_experts":11.1875,"grad-absmax/layer_01/gated_delta::out_norm":0.00014148815535008907,"hidden-states-absmax/_layer_49_pre_mlp_layernorm":6.21875,"grad-absmax/layer_37/moe::router":2.3180014977697283e-05,"hidden-states-rms/_layer_22_moe::shared_experts":0.1957923452287418,"grad-absmax/layer_33/moe::router":0.0001005215963232331,"grad-absmax/layer_08/gated_delta::out_proj":1.064736170519609e-05,"hidden-states-absmax/_layer_31_gated_delta::beta":11.5625,"hidden-states-rms/_layer_39_gated_delta::key":0.11331504138378415,"update-absmax/_layer_07_gated_delta::A_log":0.0017321407794952393,"hidden-states-l1/_layer_15_gated_delta::key":0.06164082502017319,"param-rms/_layer_12_gated_delta::out_norm":0.5578421491335291,"hidden-states-l1/_layer_34_gated_delta::key":0.13180521211686277,"update-mean/_layer_22_attention::o_proj":1.2786068737113965e-07,"grad-absmax/layer_56/moe::expert_fc2":1.4040466567166732e-06,"hidden-states-absmax/_layer_20_input_layernorm":18.125,"hidden-states-l1/_layer_10_residual_attn":0.31340212385679767,"param-l1/_layer_15_gated_delta::out_proj":0.03455391153693199,"grad-absmax/layer_22/attention::linear_qkv":0.0005274077993817627,"grad-absmax/attention::o_proj":5.995315223117359e-05,"grad-rms/layer_07/gated_delta::conv1d":9.392754422660262e-06,"hidden-states-absmax/_layer_08_gated_delta::core_attention":0.091796875,"hidden-states-l1/_layer_08_pre_mlp_layernorm":0.061245735585181184,"param-mean/_layer_17_gated_delta::A_log":0.7092684507369995,"hidden-states-rms/_layer_01_gated_delta::key":0.1894489479441035,"grad-absmax/layer_32/moe::router":0.00010129384463652968,"hidden-states-mean/_layer_15_residual_mlp":-0.0006038169514281394,"hidden-states-absmax/_layer_09_mlp":14.9375,"update-absmax/_layer_40_gated_delta::out_proj":0.00390625,"hidden-states-l1/_layer_48_gated_delta::g":3.3640467362459585,"hidden-states-absmax/_layer_43_gated_delta::key":11.0625,"hidden-states-l1/_layer_25_residual_attn":1.183841369609996,"hidden-states-rms/_layer_42_gated_delta::g":4.513704681918114,"grad-absmax/layer_37/moe::expert_fc1":5.495689947565552e-06,"param-absmax/_layer_08_gated_delta::out_norm":0.58984375,"moe/layer_09/act_rms_min_to_median":0.3045336725892851,"hidden-states-mean/_layer_11_gated_delta::in_proj":-0.18015613604473876,"param-rms/_layer_35_gated_delta::out_proj":0.044520638690672354,"moe/layer_00/act_rms_median":0,"moe/layer_52/act_rms_median":0.11161559075117111,"grad-rms/layer_25/gated_delta::dt_bias":1.499684858607052e-06,"hidden-states-absmax/_layer_29_gated_delta::beta":9.625,"param-absmax/_layer_19_gated_delta::conv1d":0.68359375,"hidden-states-rms/_layer_39_gated_delta::gate":1.3440327815399937,"grad-rms/layer_01/gated_delta::A_log":4.359159033915314e-05,"hidden-states-rms/_layer_31_gated_delta::g":6.181540096594286,"hidden-states-rms/_layer_40_residual_attn":7.564461896699379,"hidden-states-absmax/_layer_55_gated_delta::gate":27.25,"update-l1/_layer_46_gated_delta::dt_bias":0.0004523948300629854,"moe/layer_30/act_rms_median":0.1170642301440239,"param-absmax/_layer_46_gated_delta::A_log":2.3685877323150635,"hidden-states-l1/_layer_53_input_layernorm":0.27644417005111815,"hidden-states-l1/attention::o_proj":0.5024205768439436,"update-mean/_layer_27_gated_delta::out_proj":-1.5723726676242222e-07,"hidden-states-absmax/_layer_08_gated_delta::value":3.625,"update-mean/_layer_12_gated_delta::A_log":7.160077802836895e-05,"hidden-states-absmax/_layer_26_gated_delta::gate":12.9375,"hidden-states-rms/_layer_37_pre_mlp_layernorm":0.20128957117032079,"grad-absmax/layer_19/moe::router":4.536263440968469e-05,"update-mean/_layer_34_gated_delta::out_norm":0.000213623046875,"grad-absmax/layer_24/attention::o_proj":3.141237539239228e-05,"update-l1/_layer_41_gated_delta::dt_bias":0.0004936752375215292,"grad-absmax/layer_03/gated_delta::out_proj":1.2349401913525071e-05,"update-absmax/_layer_37_gated_delta::conv1d":0.0030517578125,"param-l1/_layer_07_gated_delta::conv1d":0.03507506474852562,"grad-rms/layer_20/gated_delta::out_proj":8.968465476803013e-07,"update-mean/_layer_02_gated_delta::conv1d":4.9801706154539715e-06,"param-mean/_layer_30_gated_delta::out_norm":0.734710693359375,"update-mean/_layer_28_gated_delta::out_norm":6.103515625e-05,"param-mean/_layer_19_gated_delta::out_proj":-3.122241469100118e-05,"update-mean/_layer_18_gated_delta::conv1d":4.774066383106401e-06,"hidden-states-absmax/_layer_36_gated_delta::out_norm":122.5,"grad-absmax/layer_10/moe::router":4.519302456174046e-05,"grad-rms/layer_48/moe::shared_experts":5.599645734681369e-06,"param-absmax/_layer_56_gated_delta::A_log":2.603937864303589,"hidden-states-mean/_layer_12_gated_delta::value":0.008519256571740696,"update-mean/_layer_54_gated_delta::dt_bias":8.92304815351963e-05,"update-mean/_layer_51_gated_delta::conv1d":-5.857971245859517e-06,"hidden-states-l1/_layer_13_moe::routed_experts":0.02682522360061031,"update-mean/_layer_44_gated_delta::conv1d":2.5710760382935405e-06,"param-rms/_layer_10_gated_delta::A_log":1.3052001676839335,"moe/layer_55/act_rms_min_to_median":0.3519310974935567,"hidden-states-l1/_layer_23_moe::router":0.010258910945480815,"hidden-states-mean/_layer_43_gated_delta::in_proj":-0.12653169528647484,"update-absmax/_layer_02_mlp::linear_fc1":0.00390625,"hidden-states-rms/_layer_15_mlp":0.12775393085911269,"loss-scale":1,"hidden-states-mean/_layer_56_pre_mlp_layernorm":0.00028397521711776756,"param-rms/_layer_30_gated_delta::out_norm":0.7792865876783207,"param-absmax/_layer_29_gated_delta::out_norm":0.890625,"hidden-states-absmax/_layer_28_gated_delta::in_proj":20.625,"update-mean/_layer_25_gated_delta::conv1d":5.22823120263638e-06,"hidden-states-rms/_layer_24_attention::core_attention":0.4745738295091499,"grad-absmax/layer_42/moe::shared_expert_fc2":1.0405236935184803e-05,"hidden-states-mean/_layer_50_residual_attn":-0.11150292146240728,"param-rms/_layer_45_gated_delta::dt_bias":0.4962190334689942,"hidden-states-rms/pre_mlp_layernorm":0.18063422887319225,"hidden-states-l1/_layer_42_moe::shared_experts":0.1660254348546785,"param-mean/_layer_12_gated_delta::A_log":0.9961561560630798,"moe/layer_16/act_rms_median":0.04946223646402359,"hidden-states-absmax/_layer_31_gated_delta::key":6.9375,"hidden-states-l1/_layer_28_gated_delta::query":0.033755667806347346,"moe/layer_00/act_rms_min_to_median":0,"hidden-states-rms/_layer_09_gated_delta::core_attention":0.003491770592512709,"hidden-states-rms/_layer_12_moe::shared_experts":0.13997259234359227,"grad-rms/layer_20/gated_delta::dt_bias":4.4557643103563155e-06,"grad-rms/layer_09/gated_delta::out_norm":2.2406804539303114e-05,"hidden-states-l1/_layer_36_gated_delta::value":0.04382117614998533,"hidden-states-mean/_layer_40_residual_mlp":-0.08727404770541759,"hidden-states-absmax/_layer_05_gated_delta::core_attention":0.1533203125,"update-absmax/_layer_16_gated_delta::out_norm":0.00390625,"moe/layer_54/act_rms_min_to_median":0.47521989458898983,"hidden-states-absmax/_layer_14_gated_delta::core_attention":0.12353515625,"hidden-states-l1/_layer_47_moe::shared_experts":0.37427695086919677,"hidden-states-rms/_layer_45_gated_delta::in_proj":2.4071521333500674,"param-absmax/_layer_15_gated_delta::out_norm":1.171875,"grad-absmax/layer_24/moe::shared_expert_fc1":6.384081643773243e-05,"moe/layer_50/act_rms_min_to_median":0.25880417807586,"grad-absmax/layer_08/gated_delta::conv1d":0.00011082502896897495,"hidden-states-mean/_layer_34_gated_delta::beta":-0.6496664464076097,"hidden-states-mean/_layer_53_mlp":-0.01678377589986037,"param-rms/_layer_56_gated_delta::conv1d":0.05181882968358603,"hidden-states-l1/_layer_30_pre_mlp_layernorm":0.13758492619811608,"update-rms/_layer_28_gated_delta::out_proj":0.0002003891492090136,"hidden-states-absmax/_layer_08_gated_delta::query":3.765625,"hidden-states-mean/_layer_19_gated_delta::query":0.008330280954275245,"hidden-states-l1/_layer_30_gated_delta::in_proj":0.7040956948452886,"hidden-states-absmax/_layer_24_pre_mlp_layernorm":7.375,"grad-rms/layer_29/moe::shared_expert_fc2":4.2426509716610917e-07,"grad-rms/layer_18/moe::shared_expert_fc2":2.0818654553014898e-07,"grad-rms/layer_18/moe::expert_fc1":2.0384200484653604e-08,"hidden-states-l1/_layer_20_gated_delta::beta":1.8194787287090692,"param-l1/_layer_11_gated_delta::out_norm":0.29107666015625,"update-mean/_layer_53_gated_delta::out_norm":6.103515625e-05,"hidden-states-rms/_layer_40_gated_delta::key":0.22711027986461818,"update-mean/_layer_46_gated_delta::dt_bias":0.00011629634536802769,"hidden-states-l1/_layer_34_moe::routed_experts":0.07885264788573358,"update-mean/_layer_03_gated_delta::A_log":-5.366167169995606e-05,"param-mean/_layer_39_gated_delta::dt_bias":0.392092764377594,"hidden-states-l1/_layer_53_gated_delta::g":4.881124601498199,"grad-absmax/layer_31/moe::shared_experts":6.459410542447586e-06,"hidden-states-rms/_layer_43_gated_delta::beta":1.3577311801344456,"grad-rms/layer_17/moe::router":4.280650479299595e-07,"hidden-states-l1/_layer_01_gated_delta::value":0.17150131422296933,"param-absmax/gated_delta::out_proj":3.34375,"grad-rms/layer_31/gated_delta::conv1d":8.541368743948335e-06,"hidden-states-mean/_layer_17_gated_delta::g":-6.372240826158622,"hidden-states-mean/_layer_16_moe::shared_experts":0.00010177532871229768,"hidden-states-mean/_layer_25_residual_attn":-0.013569482357142252,"param-rms/_layer_51_gated_delta::A_log":1.3472609672786622,"hidden-states-rms/_layer_45_gated_delta::query":0.9613376140580333,"hidden-states-mean/_layer_33_gated_delta::out_proj":-0.003457497519030871,"hidden-states-absmax/_layer_47_mlp":31.25,"grad-absmax/layer_26/gated_delta::A_log":5.4068732424639165e-06,"hidden-states-absmax/_layer_33_gated_delta::gate":27.375,"grad-absmax/mlp::linear_fc2":8.892357072909363e-06,"hidden-states-mean/_layer_41_moe::router":0.010099385297200726,"hidden-states-mean/_layer_25_input_layernorm":0.001502012198099458,"param-mean/_layer_48_gated_delta::A_log":1.248834490776062,"hidden-states-absmax/_layer_42_moe::routed_experts":31.875,"hidden-states-absmax/_layer_51_gated_delta::in_proj":113,"hidden-states-mean/_layer_06_gated_delta::key":-0.008511175484797417,"grad-absmax/layer_04/gated_delta::A_log":1.931060614879243e-05,"moe/layer_49/act_rms_median":0.22019381821155548,"moe/layer_29/act_rms_max":0.2677331268787384,"grad-absmax/layer_05/moe::shared_experts":2.7833118565467885e-06,"hidden-states-rms/_layer_42_pre_mlp_layernorm":0.22831689448702738,"grad-absmax/layer_03/moe::expert_fc2":2.7273983960185433e-06,"hidden-states-l1/_layer_04_gated_delta::gate":0.6810172415992309,"hidden-states-absmax/_layer_43_gated_delta::core_attention":0.30859375,"hidden-states-l1/_layer_56_gated_delta::gate":2.1948812240220574,"hidden-states-l1/_layer_36_gated_delta::out_norm":0.11820272275241885,"hidden-states-mean/_layer_35_moe::shared_experts":-0.003310610469205435,"hidden-states-absmax/_layer_28_moe::router":0.9987795948982239,"hidden-states-l1/_layer_20_gated_delta::in_proj":0.776812685442577,"hidden-states-l1/_layer_24_moe::router":0.01023259052357523,"hidden-states-l1/_layer_29_gated_delta::gate":1.0353698772632929,"hidden-states-absmax/_layer_32_residual_attn":502,"hidden-states-rms/_layer_54_gated_delta::g":6.33750969553126,"param-l1/_layer_09_gated_delta::A_log":1.2055506706237793,"param-l1/_layer_52_gated_delta::out_proj":0.03376555070281029,"update-rms/_layer_27_gated_delta::out_proj":0.00021024397645723415,"hidden-states-rms/_layer_05_gated_delta::in_proj":1.166086339566085,"hidden-states-l1/_layer_54_gated_delta::beta":1.8772766443820768,"moe/layer_07/act_rms_median":0.03006657585501671,"param-mean/_layer_07_gated_delta::out_proj":-1.1776497558457777e-05,"hidden-states-rms/_layer_39_mlp":0.3842603819083118,"grad-absmax/layer_14/moe::shared_expert_fc1":2.7498725103214383e-05,"hidden-states-l1/_layer_09_residual_attn":0.2900045859819157,"hidden-states-absmax/_layer_56_moe::routed_experts":378,"hidden-states-mean/_layer_48_mlp":-0.0013415752181053422,"param-l1/_layer_24_attention::o_proj":0.03324006497859955,"param-rms/_layer_17_gated_delta::A_log":1.118600445346481,"hidden-states-rms/_layer_56_gated_delta::core_attention":0.009114770848792328,"hidden-states-rms/_layer_10_gated_delta::core_attention":0.002146997650898961,"hidden-states-rms/_layer_15_gated_delta::value":0.050288499464164416,"hidden-states-l1/_layer_39_gated_delta::in_proj":0.9256645415190277,"hidden-states-mean/_layer_18_gated_delta::gate":0.09300243473082002,"moe/layer_01/act_rms_min":0,"hidden-states-l1/_layer_10_moe::routed_experts":0.021627122097533133,"update-mean/_layer_51_gated_delta::dt_bias":-1.2641306966543198e-05,"update-rms/_layer_56_gated_delta::out_proj":0.00017879757210257505,"_wandb":{"runtime":531833},"grad-absmax/layer_47/attention::o_proj":1.7222097085323185e-05,"grad-absmax/layer_40/moe::router":3.2862935768207535e-05,"update-absmax/_layer_17_gated_delta::A_log":0.0015902519226074219,"hidden-states-rms/_layer_10_gated_delta::query":0.11242939045281757,"hidden-states-absmax/_layer_54_input_layernorm":22.875,"hidden-states-mean/_layer_16_gated_delta::in_proj":-0.11304553825901534,"update-l1/_layer_32_gated_delta::A_log":0.0006157588213682175,"param-l1/_layer_45_gated_delta::out_norm":0.85382080078125,"hidden-states-mean/_layer_45_gated_delta::out_proj":0.0067205027306679165,"hidden-states-rms/_layer_00_embeddings":0.28919024013560396,"hidden-states-l1/_layer_50_attention::k_layernorm":2.798955058926947,"update-l1/_layer_38_gated_delta::out_proj":0.00014584707969333977,"update-rms/_layer_08_gated_delta::A_log":0.000964752465255522,"hidden-states-rms/_layer_15_moe::router":0.07216878508256834,"param-l1/_layer_33_gated_delta::conv1d":0.042806997895240784,"update-mean/_layer_18_gated_delta::dt_bias":0.00011457904474809766,"hidden-states-rms/_layer_13_moe::shared_experts":0.08974438869549729,"param-rms/_layer_34_gated_delta::A_log":1.1762851383097026,"update-absmax/_layer_25_gated_delta::A_log":0.0014252662658691406,"param-absmax/_layer_35_gated_delta::out_norm":1.9453125,"hidden-states-absmax/_layer_06_pre_mlp_layernorm":2.03125,"hidden-states-l1/_layer_38_gated_delta::key":0.051890573130914344,"hidden-states-rms/_layer_04_residual_attn":0.5624527451855534,"param-rms/_layer_29_gated_delta::conv1d":0.07846533733024098,"hidden-states-mean/_layer_12_gated_delta::out_proj":-4.513942796303591e-05,"grad-absmax/layer_49/moe::shared_expert_fc2":1.2446756045392249e-05,"update-l1/_layer_20_gated_delta::dt_bias":0.0005559741402976215,"param-rms/_layer_44_gated_delta::out_norm":1.0581733460036526,"hidden-states-mean/_layer_01_gated_delta::out_norm":-0.006627289437030487,"hidden-states-mean/_layer_04_gated_delta::out_norm":-0.00014012721873473747,"hidden-states-absmax/_layer_45_gated_delta::in_proj":45.25,"update-rms/_layer_12_gated_delta::out_proj":0.00020209381778362807,"grad-absmax/layer_02/gated_delta::A_log":9.78667685558321e-06,"hidden-states-l1/_layer_23_residual_attn":0.896994697689378,"param-l1/_layer_25_gated_delta::out_norm":0.681243896484375,"param-l1/_layer_05_gated_delta::conv1d":0.034181635826826096,"hidden-states-absmax/_layer_40_mlp":29.375,"hidden-states-mean/_layer_34_residual_mlp":-0.05042089005361677,"hidden-states-mean/_layer_48_gated_delta::gate":-0.11581562349982542,"hidden-states-absmax/_layer_40_gated_delta::in_proj":37,"param-l1/_layer_25_gated_delta::A_log":0.9152563810348511,"hidden-states-rms/_layer_19_gated_delta::out_norm":0.09859439481373519,"update-mean/_layer_38_gated_delta::dt_bias":-0.00022309855557978153,"hidden-states-rms/_layer_52_gated_delta::value":0.19613872512946254,"hidden-states-mean/_layer_29_gated_delta::query":0.03855766815634586,"moe/layer_20/act_rms_min_to_median":0.30740850662083996,"update-rms/_layer_42_gated_delta::dt_bias":0.0005440419429098719,"hidden-states-absmax/_layer_13_attention::q_layernorm":26.625,"hidden-states-rms/_layer_10_pre_mlp_layernorm":0.09140976522859007,"update-mean/_layer_55_gated_delta::dt_bias":0.0001597367227077484,"grad-rms/layer_55/gated_delta::out_proj":5.21135796866582e-07,"moe/layer_16/act_rms_min_to_median":0.3115868547326299,"hidden-states-absmax/attention::k_layernorm":42.5,"grad-absmax/layer_38/moe::shared_experts":9.043316822499037e-06,"hidden-states-mean/_layer_08_gated_delta::out_norm":-3.686200655609075e-05,"hidden-states-absmax/_layer_13_attention::core_attention":8.0625,"hidden-states-mean/_layer_15_gated_delta::query":0.008871086999307514,"hidden-states-rms/_layer_03_moe::shared_experts":0.03747635079231507,"hidden-states-absmax/_layer_56_gated_delta::query":9.625,"grad-absmax/layer_35/gated_delta::out_proj":2.1552483303821646e-05,"param-rms/_layer_31_gated_delta::out_norm":1.0042253870171216,"param-l1/_layer_01_mlp::linear_fc2":0.031317517161369324,"param-l1/_layer_23_gated_delta::A_log":0.9154301881790161,"param-rms/_layer_00_embedding":0.3697657262631521,"hidden-states-mean/_layer_36_gated_delta::g":-4.527037206535006,"grad-absmax/layer_25/moe::shared_experts":1.0484247468411922e-05,"hidden-states-mean/_layer_36_gated_delta::core_attention":0.00027341404696646894,"hidden-states-l1/_layer_43_mlp":0.3737284986649154,"update-mean/_layer_08_gated_delta::out_norm":3.0517578125e-05,"hidden-states-l1/_layer_42_pre_mlp_layernorm":0.1634984360335393,"hidden-states-absmax/_layer_01_gated_delta::out_proj":11.875,"grad-rms/layer_13/attention::linear_qkv":2.2093285416857103e-06,"param-absmax/_layer_20_gated_delta::out_proj":2.296875,"hidden-states-mean/_layer_37_input_layernorm":0.0007932679509169401,"grad-rms/layer_56/gated_delta::out_proj":1.4589441656488558e-06,"hidden-states-mean/_layer_01_gated_delta::gate":-0.8454658800918913,"hidden-states-l1/_layer_46_residual_attn":3.3268932493797836,"hidden-states-absmax/_layer_55_mlp":138,"update-mean/_layer_48_gated_delta::dt_bias":0.00019018538296222687,"grad-absmax/layer_16/gated_delta::A_log":9.778205821930896e-06,"hidden-states-rms/_layer_51_gated_delta::out_proj":16.892355906424253,"hidden-states-l1/_layer_29_gated_delta::out_proj":0.2407321024813774,"param-l1/_layer_08_gated_delta::out_proj":0.03702569380402565,"param-mean/_layer_41_gated_delta::dt_bias":0.3478732109069824,"param-mean/_layer_13_attention::linear_qkv":0.0001531672169221565,"hidden-states-mean/_layer_26_gated_delta::value":-0.0015282146126138702,"update-mean/_layer_05_gated_delta::conv1d":1.0267735888191964e-05,"param-rms/_layer_23_gated_delta::out_norm":0.7663557885654784,"hidden-states-absmax/_layer_30_moe::shared_experts":69,"hidden-states-rms/_layer_04_input_layernorm":0.3308606071347875,"update-l1/_layer_17_gated_delta::A_log":0.000461750925751403,"hidden-states-mean/_layer_06_gated_delta::in_proj":-0.19049908819307756,"update-l1/_layer_52_gated_delta::out_proj":0.0001388110249536112,"hidden-states-absmax/_layer_53_mlp":300,"hidden-states-rms/_layer_17_moe::shared_experts":0.08273741705832405,"param-rms/_layer_17_gated_delta::conv1d":0.08462242674789103,"param-rms/_layer_33_gated_delta::out_norm":0.434747072768269,"grad-rms/layer_30/gated_delta::A_log":2.3292131818985113e-06,"hidden-states-mean/_layer_10_pre_mlp_layernorm":-0.00018921329392520861,"param-mean/_layer_43_gated_delta::out_proj":2.1582389308605343e-05,"update-mean/_layer_29_gated_delta::dt_bias":1.1615193216130137e-05,"hidden-states-l1/_layer_55_residual_attn":6.4928501239836365,"param-absmax/_layer_36_gated_delta::conv1d":0.470703125,"hidden-states-absmax/_layer_55_gated_delta::in_proj":31.75,"hidden-states-l1/_layer_55_moe::router":0.01035015139085474,"grad-rms/layer_46/gated_delta::dt_bias":8.171062838812816e-06,"hidden-states-rms/_layer_11_moe::router":0.07216878508256834,"update-l1/_layer_56_gated_delta::out_proj":0.0001205386797664687,"hidden-states-l1/_layer_01_gated_delta::out_proj":0.12426786861036641,"hidden-states-l1/_layer_45_gated_delta::query":0.19596686315141476,"grad-rms/layer_22/moe::expert_fc1":1.9720456179019977e-08,"update-l1/_layer_03_gated_delta::out_proj":0.0001552809844724834,"moe/layer_37/act_rms_min_to_median":0.541293478535902,"hidden-states-rms/_layer_11_residual_attn":0.8954072553782007,"hidden-states-mean/_layer_10_input_layernorm":0.0019841005596936416,"param-mean/embedding":0.0014617833076044917,"hidden-states-rms/_layer_52_gated_delta::in_proj":3.000486876032006,"param-mean/_layer_09_gated_delta::dt_bias":0.14882813394069672,"hidden-states-mean/_layer_24_residual_attn":-0.005055757842154246,"hidden-states-rms/_layer_07_pre_mlp_layernorm":0.07573120547920471,"update-absmax/_layer_56_gated_delta::dt_bias":0.0016292333602905273,"hidden-states-absmax/_layer_44_moe::routed_experts":66.5,"moe/layer_12/act_rms_min_to_median":0.3604905150891781,"grad-rms/layer_48/gated_delta::conv1d":0.00029527168048705944,"hidden-states-rms/_layer_13_moe::router":0.07216878508256834,"grad-absmax/layer_49/attention::linear_qkv":0.00024319224758073688,"param-mean/_layer_50_attention::o_proj":-2.5028621166711673e-05,"param-rms/_layer_23_gated_delta::conv1d":0.07796808219577812,"param-mean/_layer_51_gated_delta::A_log":1.171854019165039,"param-rms/_layer_50_attention::linear_qkv":0.0400003589254533,"hidden-states-mean/_layer_53_gated_delta::g":-4.489077149519581,"param-absmax/_layer_36_gated_delta::dt_bias":0.9315854907035828,"hidden-states-mean/_layer_02_gated_delta::beta":-0.43233316330112975,"hidden-states-mean/_layer_44_gated_delta::g":-2.9717715944971554,"hidden-states-l1/_layer_28_gated_delta::core_attention":0.0010506979976681468,"hidden-states-mean/_layer_25_gated_delta::gate":0.09772327217141058,"grad-rms/layer_20/gated_delta::out_norm":1.4480540262853245e-05,"hidden-states-mean/_layer_19_gated_delta::out_proj":0.0002357218462716754,"hidden-states-mean/gated_delta::out_proj":0.0038455261643803628,"grad-rms/layer_38/gated_delta::conv1d":2.187481412237613e-05,"grad-absmax/layer_47/moe::shared_expert_fc1":3.258399374317378e-05,"hidden-states-absmax/_layer_43_mlp":37,"hidden-states-mean/_layer_36_gated_delta::value":0.009073187747381656,"hidden-states-absmax/_layer_48_gated_delta::out_proj":59.5,"hidden-states-rms/_layer_24_residual_mlp":2.4283035580060535,"update-absmax/_layer_54_gated_delta::out_proj":0.00390625,"update-l1/_layer_14_gated_delta::conv1d":0.0006495901034213603,"param-rms/_layer_20_gated_delta::A_log":1.2708571801788386,"hidden-states-rms/_layer_26_gated_delta::g":5.301612154771005,"hidden-states-rms/_layer_51_pre_mlp_layernorm":0.2697536844940227,"grad-absmax/layer_09/gated_delta::dt_bias":7.018434189376421e-06,"update-rms/_layer_41_gated_delta::conv1d":0.0007574084708084127,"hidden-states-rms/_layer_28_gated_delta::value":0.05115532721214701,"grad-absmax/layer_05/gated_delta::out_norm":0.00019903184147551656,"grad-absmax/layer_37/gated_delta::A_log":7.796380668878555e-05,"grad-rms/layer_09/gated_delta::out_proj":3.4800241983405813e-07,"param-l1/_layer_38_gated_delta::conv1d":0.029625479131937027,"param-mean/_layer_54_gated_delta::out_proj":2.2473916033050045e-05,"grad-rms/layer_10/moe::shared_expert_fc2":1.0674823085343566e-06,"hidden-states-absmax/_layer_29_pre_mlp_layernorm":3.71875,"hidden-states-l1/_layer_07_gated_delta::in_proj":0.7300395946775455,"hidden-states-mean/_layer_09_gated_delta::query":0.015108043266301545,"hidden-states-rms/_layer_03_gated_delta::g":4.142709130704982,"update-absmax/_layer_46_gated_delta::dt_bias":0.0012697577476501465,"grad-rms/layer_04/moe::shared_expert_fc1":1.9728152499173757e-07,"update-absmax/_layer_37_gated_delta::A_log":0.001273036003112793,"param-absmax/_layer_10_gated_delta::out_proj":0.89453125,"hidden-states-mean/_layer_22_mlp":0.0006332821967640158,"hidden-states-absmax/_layer_36_gated_delta::gate":26.375,"hidden-states-mean/_layer_06_gated_delta::out_norm":-2.4498104913632786e-05,"param-rms/_layer_41_gated_delta::out_norm":1.0341243566997294,"grad-rms/layer_28/moe::expert_fc2":2.1029526715675774e-08,"param-l1/_layer_03_gated_delta::conv1d":0.03438657522201538,"param-rms/_layer_24_attention::linear_qkv":0.03693844618105852,"grad-absmax/layer_36/moe::router":2.226415745099075e-05,"hidden-states-rms/_layer_43_moe::routed_experts":0.22857148223644136,"hidden-states-absmax/_layer_46_gated_delta::beta":10.875,"hidden-states-absmax/_layer_37_gated_delta::g":17.5,"grad-rms/layer_55/gated_delta::dt_bias":1.3906908558980062e-05,"hidden-states-mean/_layer_39_gated_delta::core_attention":-4.5324695693851494e-05,"hidden-states-l1/_layer_28_moe::router":0.010261876277029717,"hidden-states-rms/_layer_19_moe::routed_experts":0.07100271461815168,"update-rms/_layer_04_gated_delta::out_norm":0.000762328857226367,"grad-absmax/layer_50/attention::o_proj":3.0149238227750175e-05,"param-l1/_layer_02_gated_delta::dt_bias":0.43950551748275757,"param-l1/_layer_20_gated_delta::out_proj":0.03194015845656395,"grad-absmax/layer_28/moe::router":6.114079587860033e-05,"update-mean/_layer_11_gated_delta::out_norm":7.62939453125e-06,"grad-rms/layer_43/moe::expert_fc2":1.9533650869476778e-08,"hidden-states-mean/_layer_24_moe::shared_experts":-0.003434781161744137,"grad-absmax/layer_04/moe::shared_experts":1.3860169474355644e-06,"hidden-states-absmax/_layer_09_gated_delta::out_norm":40,"hidden-states-absmax/_layer_23_gated_delta::out_proj":51.5,"hidden-states-l1/_layer_41_input_layernorm":0.27672472914752116,"hidden-states-absmax/_layer_48_moe::routed_experts":46.25,"hidden-states-rms/_layer_17_moe::router":0.07216878508256834,"hidden-states-mean/_layer_51_moe::shared_experts":0.00661835153321642,"hidden-states-l1/_layer_45_gated_delta::out_proj":0.5517591299723542,"hidden-states-mean/_layer_26_mlp":-0.0008385001982823251,"update-mean/_layer_18_gated_delta::out_proj":-1.650334837677292e-07,"hidden-states-absmax/_layer_37_gated_delta::gate":15.9375,"hidden-states-mean/_layer_17_input_layernorm":0.0029147634032086617,"hidden-states-absmax/_layer_30_input_layernorm":17.625,"hidden-states-mean/_layer_18_gated_delta::key":0.018554658046920067,"hidden-states-absmax/_layer_20_moe::router":0.9995551109313965,"update-mean/_layer_07_gated_delta::A_log":2.108491025865078e-05,"hidden-states-l1/_layer_42_gated_delta::in_proj":0.8581018196484431,"hidden-states-absmax/_layer_47_residual_mlp":592,"update-l1/_layer_19_gated_delta::dt_bias":0.0003760427061934024,"grad-absmax/layer_04/moe::shared_expert_fc2":3.563537802619976e-06,"hidden-states-absmax/_layer_30_gated_delta::out_norm":32.75,"hidden-states-absmax/embeddings":3.15625,"grad-absmax/layer_07/moe::shared_expert_fc1":1.4311587619886268e-05,"hidden-states-l1/_layer_19_gated_delta::key":0.04896112370027586,"update-l1/_layer_19_gated_delta::out_norm":0.000762939453125,"hidden-states-rms/_layer_08_moe::routed_experts":0.03228027700537357,"hidden-states-absmax/_layer_26_gated_delta::query":8.5625,"param-absmax/_layer_17_gated_delta::out_proj":0.984375,"hidden-states-rms/_layer_11_moe::routed_experts":0.04216901549430032,"update-l1/_layer_18_gated_delta::out_proj":0.00014257618749979883,"param-absmax/_layer_56_gated_delta::out_norm":1.703125,"param-mean/_layer_36_gated_delta::conv1d":-0.00038021133514121175,"hidden-states-mean/_layer_54_residual_attn":0.10061984963653679,"hidden-states-l1/_layer_05_gated_delta::core_attention":0.0007516938808530726,"moe/layer_42/act_rms_median":0.19229665398597717,"hidden-states-absmax/_layer_57_final_layernorm":56.75,"hidden-states-absmax/_layer_28_gated_delta::beta":10.25,"grad-rms/layer_50/moe::shared_expert_fc1":4.6806946952158866e-07,"param-l1/_layer_37_gated_delta::conv1d":0.03310200944542885,"param-absmax/_layer_14_gated_delta::A_log":1.737149953842163,"hidden-states-mean/_layer_52_pre_mlp_layernorm":-0.0010266774393755125,"grad-rms/layer_28/moe::router":4.822922469944432e-07,"param-rms/_layer_07_gated_delta::out_proj":0.045780344472398864,"grad-absmax/layer_38/moe::shared_expert_fc2":6.63685113977408e-06,"update-l1/_layer_55_gated_delta::dt_bias":0.0005564838647842407,"grad-absmax/layer_40/moe::expert_fc1":2.160192252631532e-06,"param-rms/_layer_14_gated_delta::A_log":0.9851059697363832,"moe/layer_27/act_rms_min_to_median":0.39438580843517757,"grad-rms/layer_37/gated_delta::dt_bias":1.6966366696158396e-05,"param-absmax/_layer_45_gated_delta::A_log":2.0675148963928223,"grad-rms/layer_03/moe::shared_expert_fc1":2.2371546359016073e-07,"hidden-states-l1/_layer_19_gated_delta::query":0.04155084986100791,"param-absmax/_layer_09_gated_delta::A_log":2.344435453414917,"update-rms/_layer_38_gated_delta::dt_bias":0.000579291431208525,"hidden-states-absmax/_layer_30_gated_delta::query":14.3125,"hidden-states-absmax/_layer_56_gated_delta::in_proj":61.25,"moe/layer_43/act_rms_min_to_median":0.3302490183811354,"update-absmax/_layer_29_gated_delta::A_log":0.0021170377731323242,"param-mean/_layer_06_gated_delta::dt_bias":0.3837072551250458,"hidden-states-rms/_layer_38_gated_delta::value":0.05619506168217752,"hidden-states-rms/_layer_31_residual_mlp":5.641146224421414,"hidden-states-absmax/_layer_38_pre_mlp_layernorm":4.65625,"update-absmax/_layer_04_gated_delta::out_proj":0.001953125,"moe/layer_08/act_rms_max":0.06051580607891083,"hidden-states-absmax/_layer_16_residual_attn":494,"grad-rms/layer_55/moe::router":9.451063597999692e-07,"param-mean/_layer_23_gated_delta::A_log":0.8275996446609497,"moe/layer_20/act_rms_max":0.11875306069850922,"param-absmax/_layer_01_mlp::linear_fc2":0.76171875,"param-mean/_layer_11_gated_delta::dt_bias":0.07668066769838333,"update-rms/_layer_01_gated_delta::dt_bias":0.000752108857213477,"update-absmax/_layer_20_gated_delta::out_norm":0.0078125,"hidden-states-mean/_layer_07_mlp":5.772666878906414e-05,"hidden-states-rms/_layer_54_gated_delta::out_norm":0.7088399694796343,"grad-absmax/layer_17/moe::shared_expert_fc2":3.024917077709688e-06,"hidden-states-mean/_layer_42_gated_delta::in_proj":0.07867392350881351,"param-rms/_layer_52_gated_delta::A_log":1.4397747451714438,"grad-rms/layer_55/gated_delta::out_norm":1.8258918989473245e-05,"update-mean/gated_delta::A_log":-4.3516919968355145e-05,"hidden-states-absmax/_layer_16_pre_mlp_layernorm":2.9375,"grad-rms/layer_16/gated_delta::A_log":3.946028365789163e-06,"hidden-states-rms/_layer_47_attention::k_layernorm":5.3580651276699935,"hidden-states-rms/_layer_04_gated_delta::g":4.661022911160516,"hidden-states-l1/_layer_26_gated_delta::in_proj":0.7487935901023572,"hidden-states-l1/_layer_04_gated_delta::query":0.04009578854987485,"moe/layer_53/act_rms_max":0.47054004669189453,"update-absmax/_layer_25_gated_delta::conv1d":0.00390625,"update-l1/_layer_54_gated_delta::out_proj":0.00013153244799468666,"hidden-states-absmax/_layer_56_gated_delta::out_proj":728,"param-l1/_layer_23_gated_delta::out_proj":0.032348599284887314,"update-l1/_layer_41_gated_delta::conv1d":0.0005824263207614422,"hidden-states-absmax/_layer_43_gated_delta::gate":23.625,"hidden-states-rms/_layer_52_moe::shared_experts":1.0103802178734467,"hidden-states-l1/_layer_35_gated_delta::gate":1.140285627875999,"hidden-states-mean/_layer_37_residual_attn":-0.07840892117129816,"update-absmax/_layer_25_gated_delta::dt_bias":0.0014297750312834978,"grad-absmax/layer_03/moe::shared_experts":3.4217125630675582e-06,"param-rms/_layer_02_gated_delta::conv1d":0.05292508291788503,"param-absmax/mlp::linear_fc1":0.76171875,"param-mean/_layer_01_mlp::linear_fc2":-1.4975025806052145e-05,"update-rms/_layer_15_gated_delta::A_log":0.0007480674240245243,"param-mean/_layer_31_gated_delta::dt_bias":0.23677369952201843,"hidden-states-absmax/_layer_32_gated_delta::value":10.6875,"param-l1/_layer_14_gated_delta::dt_bias":0.21246859431266785,"grad-absmax/layer_39/moe::expert_fc2":1.627721758268308e-06,"hidden-states-l1/_layer_52_residual_attn":5.741318433441929,"param-rms/_layer_11_gated_delta::dt_bias":0.21767884496601023,"hidden-states-absmax/_layer_43_input_layernorm":31.25,"grad-absmax/layer_20/moe::shared_expert_fc2":2.5983306841226295e-05,"grad-absmax/layer_50/moe::router":0.00010612820915412158,"param-mean/_layer_01_gated_delta::dt_bias":0.541530191898346,"hidden-states-mean/_layer_09_input_layernorm":0.0022244011375339196,"grad-rms/layer_56/gated_delta::out_norm":1.9466792377393845e-05,"hidden-states-rms/_layer_20_gated_delta::in_proj":1.2683349809827085,"update-absmax/_layer_54_gated_delta::conv1d":0.00341796875,"hidden-states-l1/_layer_32_gated_delta::g":7.536755506605567,"hidden-states-l1/_layer_16_residual_mlp":0.5243160157580394,"hidden-states-mean/attention::core_attention":-0.001007818828537218,"hidden-states-rms/_layer_20_residual_mlp":1.569253762451297,"hidden-states-absmax/_layer_33_gated_delta::out_proj":30.5,"hidden-states-mean/_layer_02_gated_delta::key":0.008675887352333945,"param-l1/_layer_29_gated_delta::A_log":1.001509189605713,"hidden-states-l1/_layer_26_gated_delta::core_attention":0.0006478482581903944,"hidden-states-mean/_layer_51_gated_delta::out_norm":-0.00802270604940725,"hidden-states-rms/_layer_03_gated_delta::core_attention":0.002714980967156162,"hidden-states-absmax/_layer_13_attention::o_proj":17.25,"update-l1/_layer_16_gated_delta::conv1d":0.0006286893039941788,"update-l1/_layer_40_gated_delta::out_proj":0.00014331587590277195,"hidden-states-rms/_layer_15_gated_delta::in_proj":1.167436511760117,"grad-absmax/layer_51/gated_delta::out_proj":9.931134263752028e-05,"hidden-states-mean/_layer_24_attention::o_proj":-0.004227666688283287,"hidden-states-rms/_layer_53_moe::router":0.07216878508256834,"update-mean/_layer_55_gated_delta::out_proj":-2.7022013071587025e-08,"update-absmax/_layer_42_gated_delta::out_norm":0.0078125,"grad-absmax/layer_31/gated_delta::out_proj":9.40764948609285e-06,"param-absmax/_layer_20_gated_delta::dt_bias":0.9255006909370422,"hidden-states-l1/_layer_37_gated_delta::gate":1.047544246619254,"hidden-states-l1/_layer_16_mlp":0.09423241397528814,"moe/layer_08/act_rms_min_to_median":0.31917743056427295,"hidden-states-mean/_layer_25_residual_mlp":-0.013393689612292006,"hidden-states-rms/_layer_09_moe::shared_experts":0.053713220801753774,"moe/layer_38/act_rms_median":0.1517621874809265,"hidden-states-absmax/_layer_33_moe::router":0.9989287853240967,"param-mean/_layer_39_gated_delta::A_log":1.0915498733520508,"param-mean/_layer_38_gated_delta::out_norm":1.02032470703125,"param-rms/_layer_15_gated_delta::conv1d":0.0701772926310081,"grad-rms/layer_13/moe::shared_expert_fc2":2.3435067803389437e-07,"hidden-states-mean/_layer_11_moe::routed_experts":-6.476778899225415e-05,"hidden-states-absmax/_layer_24_moe::routed_experts":22.125,"hidden-states-mean/_layer_04_moe::router":0.01035224810874432,"update-l1/_layer_09_gated_delta::out_norm":0.00055694580078125,"hidden-states-l1/_layer_12_pre_mlp_layernorm":0.06710271592172574,"hidden-states-mean/_layer_05_moe::shared_experts":-0.00012859462081465074,"grad-rms/gated_delta::out_proj":5.445004227456135e-07,"param-rms/_layer_05_gated_delta::dt_bias":0.37891720234880794,"hidden-states-mean/_layer_44_pre_mlp_layernorm":0.00048133756999038993,"hidden-states-absmax/_layer_02_gated_delta::value":2.9375,"update-absmax/_layer_32_gated_delta::out_norm":0.00390625,"hidden-states-absmax/_layer_33_gated_delta::query":32.75,"grad-absmax/layer_17/moe::shared_experts":5.2699633670272306e-06,"hidden-states-mean/_layer_43_moe::routed_experts":-0.00016193004183526856,"update-rms/_layer_02_gated_delta::out_proj":0.00021260413656217555,"hidden-states-l1/_layer_40_gated_delta::value":0.056980445323043506,"hidden-states-mean/_layer_55_gated_delta::out_norm":0.00211305638819115,"grad-rms/layer_39/moe::expert_fc2":1.9867630707933452e-08,"grad-rms/layer_03/moe::shared_experts":2.69305428744629e-07,"param-mean/_layer_29_gated_delta::out_proj":6.535929060191847e-07,"hidden-states-absmax/_layer_04_mlp":5.0625,"param-absmax/_layer_16_gated_delta::conv1d":0.59375,"hidden-states-rms/_layer_48_gated_delta::out_proj":1.1063408649428463,"hidden-states-rms/_layer_13_pre_mlp_layernorm":0.10749148141960159,"hidden-states-l1/_layer_48_gated_delta::out_norm":0.22108310116328014,"hidden-states-mean/_layer_41_gated_delta::key":0.031310040510897696,"hidden-states-l1/_layer_46_gated_delta::in_proj":1.4189029664313202,"update-absmax/_layer_30_gated_delta::A_log":0.001016378402709961,"hidden-states-rms/_layer_38_pre_mlp_layernorm":0.209509354198792,"update-rms/_layer_51_gated_delta::out_norm":0.0016914558667664816,"grad-rms/gated_delta::conv1d":0.00010711068474809423,"hidden-states-rms/_layer_05_moe::routed_experts":0.023683091776065555,"hidden-states-mean/_layer_36_gated_delta::out_norm":-0.004804044137632148,"update-l1/_layer_21_attention::linear_qkv":0.000130229935166426,"param-absmax/_layer_42_gated_delta::dt_bias":0.8869812488555908,"grad-absmax/layer_52/gated_delta::out_norm":0.00044965121196582913,"grad-rms/layer_19/moe::router":4.1828602938958186e-07,"hidden-states-absmax/residual_mlp":2912,"hidden-states-l1/_layer_04_input_layernorm":0.17811464406426478,"grad-rms/layer_04/gated_delta::out_proj":4.2376983075815527e-07,"update-rms/_layer_15_gated_delta::out_proj":0.0002091642488272834,"grad-absmax/layer_33/moe::shared_expert_fc2":4.8304600568371825e-06,"param-l1/_layer_37_gated_delta::dt_bias":0.41780993342399597,"grad-absmax/layer_46/moe::shared_expert_fc2":7.200904656201601e-05,"hidden-states-absmax/_layer_51_moe::routed_experts":358,"moe/layer_41/act_rms_min":0.08947865664958954,"param-mean/mlp::linear_fc1":1.777642637534882e-05,"hidden-states-rms/_layer_16_pre_mlp_layernorm":0.11456155214102236,"grad-absmax/gated_delta::A_log":0.0005759023479185998,"grad-absmax/layer_30/gated_delta::A_log":7.348060080403229e-06,"param-absmax/_layer_52_gated_delta::out_norm":1.3359375,"grad-absmax/layer_17/gated_delta::dt_bias":1.1376481779734604e-05,"hidden-states-absmax/_layer_39_gated_delta::key":4.65625,"grad-absmax/layer_03/gated_delta::dt_bias":2.8636306524276733e-05,"param-absmax/_layer_44_gated_delta::conv1d":0.55078125,"hidden-states-mean/_layer_44_mlp":-0.0018272336887995826,"grad-rms/layer_01/mlp::linear_fc2":2.128527900786427e-07,"hidden-states-absmax/_layer_02_gated_delta::query":1.859375,"param-l1/_layer_41_gated_delta::dt_bias":0.3480212986469269,"param-l1/_layer_33_gated_delta::dt_bias":0.21303032338619232,"update-absmax/_layer_28_gated_delta::A_log":0.0011322498321533203,"hidden-states-l1/_layer_11_mlp":0.07791048472052631,"hidden-states-absmax/_layer_03_gated_delta::key":2.375,"update-rms/_layer_35_gated_delta::out_proj":0.00020829417706479984,"hidden-states-absmax/_layer_33_gated_delta::value":8.125,"update-rms/_layer_31_gated_delta::out_norm":0.0018593196059238058,"hidden-states-rms/_layer_01_residual_attn":0.28904326267584657,"hidden-states-rms/moe::shared_experts":0.6587929659444594,"hidden-states-absmax/_layer_17_residual_mlp":494,"grad-rms/layer_40/moe::shared_expert_fc2":1.1584990970967445e-07,"param-l1/_layer_11_gated_delta::dt_bias":0.18387605249881744,"moe/layer_43/act_rms_min":0.0664549320936203,"hidden-states-rms/_layer_46_gated_delta::gate":2.0077951640183365,"hidden-states-l1/_layer_27_moe::shared_experts":0.0884403244781881,"hidden-states-absmax/_layer_39_moe::router":0.9999058842658997,"update-rms/_layer_05_gated_delta::conv1d":0.0008434811969457451,"hidden-states-l1/_layer_40_gated_delta::key":0.13662703473890242,"param-absmax/_layer_02_gated_delta::dt_bias":0.8711555600166321,"grad-absmax/layer_03/gated_delta::conv1d":0.00045059173135086894,"hidden-states-absmax/_layer_21_moe::routed_experts":16.5,"hidden-states-mean/_layer_39_gated_delta::value":-0.002855965742434948,"hidden-states-l1/moe::router":0.010237777864664439,"param-mean/_layer_03_gated_delta::conv1d":0.0003078904701396823,"hidden-states-l1/_layer_45_moe::shared_experts":0.30972800370788617,"hidden-states-rms/_layer_08_gated_delta::query":0.11787173776837634,"hidden-states-absmax/_layer_08_moe::shared_experts":3.703125,"param-mean/_layer_20_gated_delta::out_norm":1.0211181640625,"hidden-states-mean/_layer_40_moe::routed_experts":-0.0001830871244043559,"hidden-states-mean/_layer_02_gated_delta::out_proj":0.0007998338922660223,"grad-rms/layer_17/moe::shared_expert_fc2":2.0332878601875175e-07,"hidden-states-l1/_layer_26_input_layernorm":0.1748698711240138,"grad-absmax/layer_27/moe::shared_expert_fc1":2.2302889192360453e-05,"hidden-states-rms/_layer_48_gated_delta::out_norm":0.6242269636871154,"grad-absmax/layer_18/gated_delta::out_proj":1.2694381439359859e-05,"update-mean/_layer_40_gated_delta::dt_bias":-9.627430699765682e-06,"grad-rms/layer_30/moe::expert_fc1":1.9875999218487872e-08,"param-absmax/_layer_39_gated_delta::A_log":1.9849311113357544,"hidden-states-absmax/_layer_14_gated_delta::gate":14.5,"update-mean/_layer_56_gated_delta::dt_bias":-0.00013112928718328476,"update-rms/_layer_56_gated_delta::dt_bias":0.0007218476360822414,"grad-rms/layer_54/gated_delta::dt_bias":4.0171230101079275e-06,"hidden-states-absmax/_layer_38_mlp":42,"param-rms/_layer_33_gated_delta::dt_bias":0.24405357334746355,"grad-rms/layer_50/moe::expert_fc1":1.965567984651805e-08,"update-rms/_layer_01_mlp::linear_fc1":0.00021115905920579016,"grad-rms/layer_14/moe::shared_expert_fc1":3.1897972643758233e-07,"update-absmax/_layer_10_gated_delta::dt_bias":0.0009400099515914917,"grad-absmax/layer_31/moe::expert_fc2":2.3406798845826415e-06,"hidden-states-rms/_layer_42_gated_delta::in_proj":1.3622040408678915,"update-rms/_layer_23_gated_delta::conv1d":0.0007640712508933432,"hidden-states-l1/_layer_04_moe::shared_experts":0.02639308886445628,"hidden-states-absmax/_layer_03_gated_delta::gate":14.5,"hidden-states-l1/_layer_52_residual_mlp":5.904161680240831,"hidden-states-rms/_layer_10_gated_delta::g":6.381861348324392,"hidden-states-l1/gated_delta::core_attention":0.0018734917940427555,"moe/layer_43/act_rms_median":0.20122672617435455,"hidden-states-mean/_layer_52_gated_delta::out_norm":-0.019553212917234053,"grad-rms/layer_06/moe::shared_expert_fc1":2.394106337078127e-07,"hidden-states-l1/_layer_42_residual_mlp":2.746652692157351,"param-rms/_layer_40_gated_delta::dt_bias":0.3467415982695141,"hidden-states-rms/_layer_54_gated_delta::out_proj":2.3249268911333165,"hidden-states-absmax/_layer_32_gated_delta::query":59,"param-l1/_layer_44_gated_delta::out_norm":1.04547119140625,"param-absmax/_layer_53_gated_delta::dt_bias":1.0506173372268677,"grad-rms/layer_33/moe::expert_fc2":2.3495882605067765e-08,"hidden-states-mean/_layer_37_pre_mlp_layernorm":0.0005370713246475249,"hidden-states-rms/_layer_51_moe::routed_experts":0.5333673087475852,"hidden-states-absmax/_layer_37_gated_delta::out_proj":33.25,"hidden-states-absmax/_layer_46_gated_delta::out_norm":233,"hidden-states-rms/_layer_02_mlp":0.11324608454934386,"hidden-states-l1/_layer_47_attention::k_layernorm":3.919395914005746,"grad-rms/layer_05/gated_delta::out_proj":5.31475904619455e-07,"param-absmax/_layer_34_gated_delta::out_norm":1.3125,"hidden-states-mean/_layer_25_gated_delta::core_attention":0.0003961236688344084,"update-l1/_layer_46_gated_delta::conv1d":0.0005559714045375586,"param-absmax/_layer_50_attention::linear_qkv":0.6015625,"hidden-states-absmax/_layer_36_gated_delta::key":3.53125,"hidden-states-mean/_layer_17_gated_delta::value":0.0034228599985373018,"hidden-states-rms/_layer_32_gated_delta::gate":1.809821526551561,"hidden-states-rms/gated_delta::out_norm":0.5581695541127467,"param-mean/_layer_14_gated_delta::dt_bias":-0.16833387315273285,"grad-rms/layer_15/gated_delta::out_norm":1.6221609673659274e-05,"update-rms/_layer_45_gated_delta::A_log":0.0006040633834868595,"hidden-states-rms/_layer_26_moe::shared_experts":0.13652899511817707,"hidden-states-l1/_layer_28_residual_mlp":1.409888674007496,"moe/layer_14/act_rms_median":0.04452597722411156,"hidden-states-l1/_layer_47_moe::router":0.01022758497438192,"grad-rms/layer_51/gated_delta::dt_bias":6.171641620996476e-06,"update-rms/_layer_03_gated_delta::dt_bias":0.0007166889058011181,"grad-absmax/layer_52/moe::shared_expert_fc2":3.215754622942768e-05,"param-l1/_layer_09_gated_delta::out_norm":0.36493682861328125,"grad-absmax/layer_07/moe::router":4.727229315903969e-05,"hidden-states-mean/_layer_40_gated_delta::key":-0.0040317257084519515,"grad-rms/layer_31/moe::shared_expert_fc1":2.361390108099212e-07,"hidden-states-rms/residual_mlp":10.229597692336533,"update-l1/_layer_02_gated_delta::out_proj":0.0001493030576966703,"hidden-states-absmax/_layer_56_moe::router":0.999609649181366,"grad-rms/layer_36/moe::shared_experts":2.066993507093513e-06,"hidden-states-absmax/gated_delta::out_norm":832,"grad-absmax/layer_52/moe::router":7.099409413058311e-05,"hidden-states-absmax/_layer_34_gated_delta::g":19.625,"param-rms/_layer_14_gated_delta::out_proj":0.044435650390146283,"param-l1/_layer_16_gated_delta::A_log":0.920375406742096,"grad-rms/layer_26/moe::expert_fc2":2.2019216575181048e-08,"grad-rms/layer_11/gated_delta::dt_bias":2.473665166046519e-06,"moe/layer_01/act_rms_min_to_median":0,"hidden-states-rms/_layer_33_gated_delta::core_attention":0.007972512877450719,"hidden-states-mean/_layer_32_gated_delta::core_attention":0.0003309110574020216,"hidden-states-mean/_layer_43_residual_attn":-0.09321707853282434,"grad-absmax/layer_38/moe::router":4.6508612285833806e-05,"grad-absmax/layer_30/gated_delta::out_proj":1.583355333423242e-05,"hidden-states-mean/_layer_56_gated_delta::g":-2.819355347079676,"param-mean/_layer_26_gated_delta::out_proj":-4.922101288684644e-05,"grad-absmax/layer_30/gated_delta::dt_bias":7.1646036303718574e-06,"grad-absmax/layer_55/moe::expert_fc2":9.758119858815917e-07,"hidden-states-rms/_layer_25_residual_mlp":2.936951619068081,"grad-absmax/layer_45/moe::expert_fc1":4.24100744567113e-06,"hidden-states-absmax/_layer_15_gated_delta::out_norm":11.9375,"hidden-states-absmax/_layer_15_gated_delta::key":5.875,"grad-rms/layer_20/gated_delta::conv1d":3.164153424075847e-05,"moe/layer_36/act_rms_max":0.29071444272994995,"hidden-states-absmax/_layer_54_mlp":54,"hidden-states-mean/_layer_06_moe::shared_experts":4.830800965630937e-05,"grad-absmax/gated_delta::out_proj":0.00026976419030688703,"hidden-states-rms/_layer_34_gated_delta::gate":1.6650772501120181,"update-rms/_layer_41_gated_delta::A_log":0.0006174284702710821,"update-mean/_layer_07_gated_delta::out_norm":-4.57763671875e-05,"param-rms/_layer_29_gated_delta::out_proj":0.04389324414957319,"hidden-states-rms/_layer_05_gated_delta::g":4.197159992465466,"hidden-states-l1/_layer_18_moe::routed_experts":0.03676624081496326,"hidden-states-l1/_layer_48_moe::shared_experts":0.33113719000067593,"update-rms/_layer_25_gated_delta::conv1d":0.000829274211139352,"hidden-states-rms/_layer_19_gated_delta::key":0.10714670600864247,"hidden-states-mean/_layer_27_gated_delta::key":0.0026192128055349357,"param-absmax/_layer_24_attention::o_proj":0.97265625,"hidden-states-l1/_layer_52_pre_mlp_layernorm":0.1528138104455245,"param-mean/_layer_38_gated_delta::dt_bias":0.4427039921283722,"hidden-states-mean/_layer_44_moe::shared_experts":-0.0006801031642157631,"param-mean/_layer_16_gated_delta::A_log":0.7200564742088318,"hidden-states-rms/_layer_28_gated_delta::in_proj":1.440036538688275,"hidden-states-mean/_layer_16_gated_delta::core_attention":6.895824575485238e-05,"hidden-states-mean/_layer_05_gated_delta::core_attention":5.062279036358666e-05,"hidden-states-mean/_layer_16_pre_mlp_layernorm":-0.0003996177878240839,"moe/layer_06/act_rms_min":0.006749803200364113,"moe/layer_23/act_rms_median":0.1033167615532875,"hidden-states-rms/_layer_06_gated_delta::beta":2.0241344561594277,"hidden-states-absmax/_layer_23_moe::routed_experts":8.8125,"hidden-states-mean/_layer_15_pre_mlp_layernorm":-0.0003267567761473985,"grad-absmax/layer_24/attention::linear_qkv":0.0005793650634586811,"hidden-states-absmax/_layer_53_gated_delta::out_proj":169,"update-l1/_layer_26_gated_delta::dt_bias":0.0005389837897382677,"grad-absmax/gated_delta::dt_bias":0.0005705449730157852,"param-absmax/_layer_07_gated_delta::out_proj":0.94921875,"hidden-states-absmax/_layer_26_gated_delta::value":2.546875,"param-l1/_layer_31_gated_delta::out_proj":0.03597918525338173,"hidden-states-l1/_layer_03_moe::shared_experts":0.02515251303047097,"hidden-states-mean/_layer_15_gated_delta::gate":-0.17831589917134036,"grad-absmax/layer_07/gated_delta::out_norm":7.745358016109094e-05,"param-l1/_layer_17_gated_delta::out_norm":0.620819091796875,"hidden-states-l1/_layer_46_gated_delta::out_proj":0.7307471908029546,"grad-absmax/layer_12/moe::router":5.178853825782426e-05,"update-absmax/_layer_57_lm_head":0.0078125,"hidden-states-mean/_layer_04_residual_attn":0.0016261273605230285,"hidden-states-absmax/_layer_05_mlp":12.125,"hidden-states-rms/_layer_53_gated_delta::query":0.1987993019425278,"moe/layer_47/act_rms_min_to_median":0.33066552752937267,"param-absmax/_layer_28_gated_delta::conv1d":0.416015625,"hidden-states-rms/_layer_25_gated_delta::value":0.057745849901054316,"param-mean/_layer_48_gated_delta::out_norm":1.5618896484375,"update-rms/_layer_05_gated_delta::out_proj":0.0002199291614305157,"grad-absmax/layer_34/moe::expert_fc2":1.926614459080156e-06,"hidden-states-rms/_layer_44_moe::routed_experts":0.23973337758064567,"grad-absmax/layer_16/gated_delta::conv1d":0.0006280373781919479,"hidden-states-mean/_layer_53_gated_delta::gate":0.098908592630457,"update-absmax/_layer_02_gated_delta::conv1d":0.003082275390625,"param-mean/_layer_55_gated_delta::A_log":1.707268476486206,"grad-absmax/layer_10/gated_delta::conv1d":0.00025895328144542873,"update-l1/_layer_01_gated_delta::out_norm":0.0003662109375,"hidden-states-mean/_layer_25_gated_delta::out_norm":-0.0031642749952405687,"grad-absmax/layer_26/moe::expert_fc1":2.5152908165182453e-06,"hidden-states-l1/_layer_27_residual_mlp":1.336079875955567,"param-mean/_layer_42_gated_delta::out_norm":1.200439453125,"param-l1/_layer_25_gated_delta::conv1d":0.045544739812612534,"hidden-states-rms/_layer_27_gated_delta::key":0.16511272019613055,"param-absmax/_layer_44_gated_delta::dt_bias":1.2071597576141357,"param-absmax/_layer_54_gated_delta::out_proj":1.0390625,"hidden-states-mean/_layer_39_residual_attn":-0.08534149352806729,"param-mean/_layer_23_gated_delta::out_norm":0.728668212890625,"hidden-states-rms/_layer_31_moe::routed_experts":0.13715657819942648,"update-mean/_layer_01_gated_delta::dt_bias":-0.00024692926672287285,"param-absmax/_layer_25_gated_delta::A_log":1.5486246347427368,"hidden-states-l1/_layer_54_pre_mlp_layernorm":0.12020860555274553,"hidden-states-absmax/_layer_46_pre_mlp_layernorm":13.4375,"hidden-states-rms/_layer_15_gated_delta::gate":1.0873826206044697,"hidden-states-absmax/_layer_30_gated_delta::value":4.5,"hidden-states-l1/_layer_03_gated_delta::key":0.03870663880784974,"hidden-states-absmax/_layer_25_residual_attn":500,"hidden-states-rms/_layer_41_gated_delta::gate":1.4410205553110682,"hidden-states-rms/_layer_18_gated_delta::in_proj":1.067832453664216,"hidden-states-rms/_layer_10_residual_mlp":0.8083149639182255,"moe/layer_08/act_rms_min":0.010233237408101559,"hidden-states-absmax/_layer_41_gated_delta::gate":20.75,"update-rms/_layer_13_attention::o_proj":0.00016596711853733963,"moe/layer_55/act_rms_median":0.07199662923812866,"grad-absmax/layer_21/attention::linear_qkv":0.0010825495701283216,"update-rms/_layer_36_gated_delta::out_norm":0.0006905339660024879,"update-mean/_layer_09_gated_delta::out_norm":0.00019073486328125,"grad-rms/layer_44/moe::expert_fc2":1.935612367944615e-08,"param-l1/_layer_08_gated_delta::A_log":1.1367273330688477,"grad-absmax/layer_06/gated_delta::dt_bias":0.0002359328791499138,"hidden-states-absmax/_layer_13_attention::k_layernorm":22.375,"grad-absmax/layer_20/gated_delta::conv1d":0.0007564747356809676,"hidden-states-l1/_layer_35_gated_delta::key":0.10740116028799537,"update-rms/_layer_40_gated_delta::dt_bias":0.0007023508657134699,"param-absmax/_layer_31_gated_delta::A_log":1.8612569570541382,"hidden-states-absmax/_layer_32_gated_delta::g":19.375,"param-rms/_layer_11_gated_delta::conv1d":0.06369725157543088,"grad-absmax/moe::shared_expert_fc1":0.00023853527090977877,"update-rms/_layer_37_gated_delta::out_proj":0.00021171636946460068,"hidden-states-mean/_layer_20_moe::router":0.010258031776830637,"hidden-states-mean/_layer_03_gated_delta::beta":-0.42179455420906,"param-l1/_layer_54_gated_delta::conv1d":0.03134450316429138,"hidden-states-rms/_layer_52_gated_delta::out_norm":0.5853564718323488,"param-rms/_layer_06_gated_delta::out_proj":0.045192422106741625,"hidden-states-mean/_layer_31_gated_delta::key":0.012558303407824446,"hidden-states-l1/_layer_36_pre_mlp_layernorm":0.14351734739748803,"param-mean/_layer_20_gated_delta::dt_bias":0.16874442994594574,"hidden-states-rms/_layer_23_gated_delta::out_norm":0.27756719899887805,"hidden-states-mean/_layer_48_gated_delta::query":0.01042051178759477,"grad-absmax/layer_50/moe::shared_expert_fc2":7.731730147497728e-05,"hidden-states-rms/_layer_53_residual_mlp":25.807293073510397,"hidden-states-absmax/_layer_11_input_layernorm":19.25,"moe/layer_20/act_rms_min":0.019687363877892494,"hidden-states-absmax/_layer_21_pre_mlp_layernorm":5.40625,"hidden-states-absmax/_layer_30_gated_delta::in_proj":16.75,"hidden-states-absmax/_layer_20_residual_mlp":494,"grad-absmax/layer_44/gated_delta::conv1d":0.005737845320254564,"hidden-states-absmax/_layer_27_input_layernorm":15.125,"hidden-states-rms/_layer_54_pre_mlp_layernorm":0.18471589066122784,"param-absmax/_layer_40_gated_delta::out_norm":1.3359375,"grad-rms/layer_25/moe::router":7.172993073420224e-07,"hidden-states-mean/_layer_04_gated_delta::query":0.007787253704637731,"hidden-states-absmax/_layer_51_gated_delta::core_attention":0.6953125,"update-rms/_layer_43_gated_delta::conv1d":0.0007834223731648204,"grad-absmax/layer_39/moe::shared_expert_fc1":1.9340781364007853e-05,"hidden-states-mean/_layer_44_gated_delta::core_attention":0.0005116590792242085,"grad-rms/layer_40/moe::expert_fc2":1.9152035831277042e-08,"hidden-states-absmax/_layer_07_moe::router":0.9936287999153137,"grad-rms/layer_50/moe::shared_expert_fc2":4.838813039641931e-07,"hidden-states-absmax/_layer_28_gated_delta::key":2.625,"hidden-states-l1/_layer_05_moe::shared_experts":0.02666923349132937,"hidden-states-rms/_layer_48_gated_delta::core_attention":0.004166099402089776,"hidden-states-l1/_layer_34_gated_delta::core_attention":0.00148033263579251,"hidden-states-rms/_layer_46_gated_delta::key":0.16628223648014612,"grad-rms/layer_03/moe::shared_expert_fc2":2.0392854007268784e-07,"param-rms/_layer_47_attention::linear_qkv":0.039225122080884786,"param-absmax/_layer_30_gated_delta::dt_bias":0.4790635406970978,"hidden-states-absmax/_layer_30_gated_delta::out_proj":28.5,"hidden-states-l1/_layer_08_gated_delta::out_norm":0.015998134434000784,"hidden-states-rms/_layer_27_moe::shared_experts":0.13375405142349858,"hidden-states-l1/_layer_08_gated_delta::core_attention":0.0002425944547188377,"update-absmax/gated_delta::A_log":0.0025861263275146484,"hidden-states-mean/_layer_46_input_layernorm":0.006469435829750994,"hidden-states-absmax/_layer_32_moe::shared_experts":24.25,"hidden-states-rms/_layer_31_gated_delta::core_attention":0.0013234670876631036,"moe/layer_45/act_rms_min_to_median":0.2969916582638587,"hidden-states-rms/_layer_23_gated_delta::out_proj":0.5286500678838755,"hidden-states-l1/_layer_30_gated_delta::query":0.06996103465092596,"hidden-states-l1/_layer_40_gated_delta::g":6.821938374273583,"update-mean/_layer_02_gated_delta::A_log":-0.00014336511958390474,"grad-absmax/layer_29/gated_delta::conv1d":0.00036773571628145874,"hidden-states-absmax/_layer_45_residual_attn":502,"hidden-states-l1/_layer_52_gated_delta::query":0.11430352053684241,"hidden-states-rms/_layer_37_residual_attn":7.192650216381779,"hidden-states-l1/_layer_17_gated_delta::out_norm":0.039857444448166426,"update-mean/_layer_29_gated_delta::out_proj":1.4883568155710236e-07,"hidden-states-rms/_layer_51_input_layernorm":0.7635356329234975,"param-absmax/_layer_37_gated_delta::out_proj":0.671875,"param-rms/_layer_01_gated_delta::dt_bias":0.6850935477749039,"hidden-states-l1/_layer_02_input_layernorm":0.19400584030220988,"hidden-states-absmax/_layer_39_pre_mlp_layernorm":3.921875,"hidden-states-mean/residual_mlp":-0.01953925374217286,"hidden-states-absmax/_layer_34_moe::shared_experts":16.25,"grad-absmax/layer_10/gated_delta::out_proj":3.255784758948721e-05,"param-mean/_layer_45_gated_delta::out_proj":3.774938886635937e-05,"hidden-states-l1/_layer_55_moe::routed_experts":0.06314196070882296,"param-absmax/gated_delta::A_log":2.8615500926971436,"update-l1/_layer_35_gated_delta::conv1d":0.0006363296415656805,"grad-rms/layer_37/gated_delta::out_norm":9.258666639356581e-06,"update-rms/_layer_32_gated_delta::conv1d":0.0008616443490479768,"hidden-states-absmax/_layer_52_pre_mlp_layernorm":13.875,"hidden-states-l1/_layer_27_gated_delta::gate":0.7377575323292651,"grad-absmax/layer_47/moe::shared_experts":1.6964277165243402e-05,"grad-absmax/layer_06/gated_delta::A_log":0.00023621809668838978,"update-l1/_layer_17_gated_delta::conv1d":0.0006401492282748222,"grad-rms/layer_42/gated_delta::out_norm":5.9470902325436795e-06,"hidden-states-rms/_layer_52_gated_delta::out_proj":1.4987566979392877,"hidden-states-rms/_layer_54_gated_delta::value":0.10919669862528802,"hidden-states-mean/_layer_40_gated_delta::out_proj":-0.0013550915042420035,"hidden-states-l1/_layer_55_input_layernorm":0.3290888165341397,"grad-absmax/layer_54/gated_delta::A_log":1.994621561607346e-05,"param-absmax/_layer_39_gated_delta::dt_bias":0.777550458908081,"grad-absmax/layer_15/gated_delta::out_norm":5.184691690374166e-05,"update-l1/_layer_11_gated_delta::out_proj":0.00014796621690038592,"hidden-states-rms/_layer_35_gated_delta::g":7.806280704267012,"update-mean/attention::linear_qkv":-1.6011961359982e-09,"hidden-states-mean/_layer_23_gated_delta::beta":-1.4051287657175977,"grad-absmax/layer_35/moe::expert_fc2":7.071758318488719e-06,"hidden-states-absmax/_layer_48_pre_mlp_layernorm":6.1875,"hidden-states-l1/_layer_33_gated_delta::beta":1.5377435133151427,"lm-token-prepare-seconds":0.042549096047878265,"param-rms/_layer_34_gated_delta::conv1d":0.06353048821652583,"hidden-states-l1/_layer_18_residual_attn":0.5662989099382054,"hidden-states-rms/_layer_29_residual_mlp":3.9765320394553276,"hidden-states-l1/_layer_15_gated_delta::gate":0.6820578196117663,"update-rms/_layer_08_gated_delta::out_proj":0.00023002766109381839,"update-mean/_layer_30_gated_delta::A_log":-0.0001263837330043316,"update-rms/_layer_10_gated_delta::dt_bias":0.0004998937477294148,"hidden-states-l1/_layer_12_gated_delta::query":0.0569123842611771,"hidden-states-absmax/_layer_32_gated_delta::gate":27,"hidden-states-l1/_layer_38_gated_delta::value":0.027420494248925953,"grad-absmax/layer_33/gated_delta::dt_bias":2.3008562948234612e-06,"hidden-states-absmax/_layer_48_gated_delta::out_norm":141,"hidden-states-absmax/_layer_15_moe::routed_experts":15.0625,"hidden-states-mean/_layer_34_mlp":-0.0022249168574710915,"hidden-states-mean/_layer_18_moe::routed_experts":-4.292195370456897e-05,"hidden-states-mean/gated_delta::beta":-0.7011556643165558,"hidden-states-rms/_layer_04_gated_delta::out_proj":0.07045037365213869,"hidden-states-rms/_layer_11_gated_delta::g":7.152804175287155,"hidden-states-mean/_layer_51_residual_attn":-0.10413881887660252,"hidden-states-l1/_layer_33_input_layernorm":0.3067396052765376,"hidden-states-mean/_layer_37_gated_delta::out_norm":0.0009638142058329827,"hidden-states-absmax/_layer_23_gated_delta::out_norm":63.5,"hidden-states-mean/_layer_30_gated_delta::out_proj":0.0008010602114123358,"hidden-states-absmax/_layer_09_gated_delta::out_proj":13.3125,"hidden-states-rms/_layer_37_gated_delta::beta":1.4295915266554138,"param-rms/_layer_14_gated_delta::dt_bias":0.276728619507734,"hidden-states-absmax/_layer_18_gated_delta::query":5.40625,"param-absmax/_layer_35_gated_delta::dt_bias":0.7267091274261475,"hidden-states-mean/_layer_43_gated_delta::gate":-0.2970459517295303,"param-l1/_layer_40_gated_delta::A_log":1.1923798322677612,"param-l1/_layer_34_gated_delta::A_log":1.0457173585891724,"grad-rms/layer_02/gated_delta::dt_bias":1.884553782011713e-06,"hidden-states-l1/_layer_43_input_layernorm":0.30601874661669093,"grad-rms/layer_10/moe::shared_expert_fc1":2.76357995168307e-07,"hidden-states-rms/_layer_17_gated_delta::value":0.06159921918096234,"grad-rms/layer_04/moe::shared_expert_fc2":1.33027386662998e-07,"update-mean/_layer_25_gated_delta::A_log":6.8876834120601416e-06,"update-rms/mlp::linear_fc2":0.00021928415151594655,"hidden-states-rms/_layer_23_pre_mlp_layernorm":0.14727043571550844,"update-absmax/_layer_10_gated_delta::out_proj":0.001953125,"update-absmax/_layer_01_mlp::linear_fc1":0.001953125,"update-rms/_layer_16_gated_delta::out_proj":0.00020527668322535965,"hidden-states-l1/_layer_35_pre_mlp_layernorm":0.1487139491693913,"hidden-states-rms/_layer_37_gated_delta::query":0.17414259561649692,"param-rms/_layer_53_gated_delta::out_proj":0.0449111345139301,"hidden-states-mean/_layer_16_gated_delta::beta":-1.4221292045270448,"update-l1/_layer_43_gated_delta::out_proj":0.0001430895645171404,"hidden-states-mean/_layer_16_residual_mlp":-0.0008035821750373751,"grad-absmax/layer_07/moe::expert_fc2":3.377940629434306e-06,"hidden-states-absmax/_layer_04_gated_delta::core_attention":0.240234375,"hidden-states-mean/_layer_09_gated_delta::key":0.013410306271649155,"moe/layer_06/act_rms_median":0.027804305776953697,"grad-absmax/layer_26/moe::shared_expert_fc2":3.687668595375726e-06,"grad-absmax/layer_41/moe::shared_expert_fc2":3.697548891068436e-05,"grad-absmax/layer_22/moe::shared_experts":6.550390025950037e-06,"update-mean/_layer_16_gated_delta::out_proj":-6.210510150594928e-08,"grad-absmax/layer_21/moe::shared_experts":1.615340806893073e-05,"hidden-states-mean/moe::router":0.010237777864664439,"hidden-states-l1/_layer_21_moe::router":0.010260928190498064,"hidden-states-rms/_layer_19_gated_delta::beta":2.884952243622663,"hidden-states-l1/_layer_49_attention::core_attention":0.32598764610143255,"update-absmax/_layer_08_gated_delta::dt_bias":0.0022715628147125244,"hidden-states-rms/_layer_38_mlp":0.36588523258096955,"hidden-states-absmax/_layer_34_gated_delta::key":7.1875,"param-mean/_layer_25_gated_delta::A_log":0.7949585914611816,"hidden-states-l1/_layer_55_gated_delta::out_norm":0.17699435973100183,"moe/layer_50/act_rms_median":0.15296031534671783,"hidden-states-l1/_layer_33_gated_delta::out_proj":0.22293544983094332,"param-mean/_layer_52_gated_delta::out_norm":0.891357421875,"param-rms/_layer_46_gated_delta::dt_bias":0.656548727255106,"update-l1/_layer_37_gated_delta::out_proj":0.00014809063577558845,"hidden-states-mean/_layer_46_gated_delta::query":0.018140288369689984,"hidden-states-absmax/_layer_34_moe::router":0.9993246793746948,"hidden-states-rms/_layer_13_attention::core_attention":0.1925426650365761,"grad-absmax/layer_04/gated_delta::dt_bias":1.90797964023659e-05,"grad-absmax/layer_05/gated_delta::dt_bias":3.5079381177638425e-06,"param-mean/_layer_25_gated_delta::conv1d":-0.00045830034650862217,"update-absmax/_layer_15_gated_delta::A_log":0.0025436878204345703,"update-mean/_layer_14_gated_delta::out_norm":6.103515625e-05,"param-mean/_layer_48_gated_delta::dt_bias":0.6855649948120117,"param-absmax/_layer_47_attention::linear_qkv":0.62109375,"hidden-states-l1/_layer_48_input_layernorm":0.2914117326035181,"batch-size":32,"update-mean/_layer_15_gated_delta::out_proj":-1.7028449406097934e-07,"update-l1/_layer_28_gated_delta::out_norm":0.00054931640625,"hidden-states-absmax/_layer_11_residual_mlp":478,"grad-rms/layer_13/moe::shared_expert_fc1":3.620194328648847e-07,"hidden-states-mean/_layer_45_moe::router":0.010212490095693286,"hidden-states-l1/_layer_56_gated_delta::key":0.08761750461071785,"param-rms/_layer_49_attention::linear_qkv":0.03916735880152264,"hidden-states-rms/_layer_28_mlp":0.2668716563166041,"hidden-states-l1/_layer_10_gated_delta::out_norm":0.02594597015630843,"hidden-states-mean/_layer_26_moe::routed_experts":-6.426892993213729e-05,"update-absmax/_layer_25_gated_delta::out_norm":0.0078125,"hidden-states-mean/_layer_39_gated_delta::key":0.01515574188896917,"hidden-states-l1/_layer_05_gated_delta::key":0.049253566474147616,"param-rms/_layer_01_gated_delta::conv1d":0.05153959535008449,"param-rms/_layer_01_mlp::linear_fc1":0.044206502691831934,"param-l1/_layer_36_gated_delta::conv1d":0.028311727568507195,"hidden-states-l1/_layer_37_gated_delta::g":2.750852674347481,"update-rms/_layer_09_gated_delta::conv1d":0.0008419396578467817,"hidden-states-l1/_layer_02_gated_delta::key":0.04022617372586414,"hidden-states-l1/_layer_06_residual_attn":0.23016234995869994,"update-l1/_layer_39_gated_delta::out_proj":0.00014260153693612665,"hidden-states-rms/_layer_48_moe::shared_experts":0.569885441491504,"hidden-states-absmax/_layer_55_input_layernorm":31.375,"grad-rms/layer_45/moe::router":8.42693024781948e-07,"param-absmax/_layer_28_gated_delta::out_norm":1.515625,"update-rms/_layer_02_gated_delta::dt_bias":0.0007412110766732004,"hidden-states-rms/_layer_54_gated_delta::in_proj":2.26174771655651,"param-absmax/_layer_03_gated_delta::conv1d":0.318359375,"hidden-states-l1/_layer_35_input_layernorm":0.28441024236801177,"hidden-states-l1/_layer_49_mlp":0.5644080119619383,"param-rms/_layer_18_gated_delta::dt_bias":0.14422720425583205,"hidden-states-absmax/_layer_56_gated_delta::out_norm":390,"hidden-states-absmax/_layer_44_gated_delta::query":11.4375,"hidden-states-l1/_layer_26_residual_attn":1.2455472046789176,"hidden-states-mean/_layer_46_residual_attn":-0.09156025547507471,"hidden-states-l1/_layer_36_gated_delta::g":4.5323920884599795,"hidden-states-rms/_layer_09_gated_delta::beta":2.3371089552641173,"hidden-states-mean/_layer_30_residual_mlp":-0.029436185739777774,"hidden-states-mean/_layer_45_gated_delta::g":-6.615098311316766,"hidden-states-mean/_layer_24_pre_mlp_layernorm":-0.0008092333246784895,"hidden-states-rms/_layer_45_pre_mlp_layernorm":0.2710987779870108,"hidden-states-l1/_layer_13_attention::q_layernorm":1.2110671056589015,"param-mean/_layer_32_gated_delta::conv1d":-0.0006203852244652808,"hidden-states-l1/gated_delta::in_proj":1.069879367842957,"hidden-states-mean/gated_delta::in_proj":-0.1387006671069023,"hidden-states-rms/_layer_28_moe::shared_experts":0.13515907136606223,"grad-absmax/gated_delta::out_norm":0.00044965121196582913,"hidden-states-l1/_layer_41_pre_mlp_layernorm":0.15968852158648034,"moe/layer_28/act_rms_max":0.24566663801670074,"hidden-states-mean/_layer_32_gated_delta::out_proj":-0.003494521786737059,"update-mean/_layer_19_gated_delta::dt_bias":-5.823648098157719e-05,"param-absmax/_layer_02_mlp::linear_fc1":0.76171875,"param-rms/_layer_35_gated_delta::A_log":1.267837004242573,"update-absmax/attention::o_proj":0.0078125,"param-absmax/_layer_31_gated_delta::conv1d":0.80859375,"hidden-states-mean/_layer_54_gated_delta::out_proj":-0.019905604932380312,"hidden-states-rms/_layer_42_gated_delta::out_norm":0.27257449595054256,"param-rms/_layer_35_gated_delta::conv1d":0.08159151459604558,"hidden-states-l1/_layer_48_gated_delta::query":0.05843300696408832,"hidden-states-absmax/_layer_33_gated_delta::out_norm":55.5,"update-l1/_layer_29_gated_delta::dt_bias":0.0005795720499008894,"grad-rms/layer_32/gated_delta::out_norm":1.8005669544891753e-05,"hidden-states-mean/_layer_40_gated_delta::core_attention":0.00014696766744541138,"hidden-states-mean/attention::q_layernorm":-0.007722794649673361,"grad-absmax/layer_48/moe::expert_fc2":2.9049003842374077e-06,"hidden-states-mean/_layer_43_gated_delta::out_proj":0.0012540328647160964,"hidden-states-l1/moe::shared_experts":0.15810249695493078,"hidden-states-l1/_layer_06_input_layernorm":0.21982529399696518,"grad-absmax/layer_32/moe::shared_experts":1.3902052160119638e-05,"update-absmax/_layer_28_gated_delta::dt_bias":0.0011283457279205322,"hidden-states-absmax/_layer_41_mlp":50.75,"grad-absmax/layer_08/gated_delta::A_log":5.350272203941131e-06,"hidden-states-l1/_layer_43_gated_delta::gate":1.1244459300572789,"hidden-states-l1/_layer_19_moe::shared_experts":0.07754957855338794,"hidden-states-mean/_layer_45_gated_delta::beta":-1.009400249648124,"hidden-states-rms/_layer_46_input_layernorm":0.7850442012180321,"hidden-states-mean/_layer_46_gated_delta::key":0.0054517562480874355,"moe/layer_49/act_rms_min_to_median":0.2585029398489563,"hidden-states-absmax/_layer_24_input_layernorm":40.75,"hidden-states-mean/_layer_01_residual_mlp":2.04605936976904e-05,"hidden-states-mean/_layer_08_gated_delta::value":-0.0011966229911593187,"hidden-states-l1/_layer_53_gated_delta::value":0.07030589971612689,"hidden-states-absmax/_layer_44_gated_delta::value":7.53125,"update-l1/gated_delta::out_proj":0.00014387625352745608,"update-absmax/_layer_56_gated_delta::out_proj":0.00390625,"update-l1/_layer_48_gated_delta::out_norm":0.000244140625,"grad-rms/layer_31/moe::shared_expert_fc2":1.5963524229208347e-07,"update-l1/_layer_14_gated_delta::A_log":0.0004622116684913635,"param-rms/_layer_15_gated_delta::out_norm":0.5389277801238815,"grad-rms/moe::expert_fc1":1.8551254098854648e-08,"hidden-states-absmax/_layer_37_gated_delta::out_norm":94,"grad-absmax/layer_12/gated_delta::out_proj":7.36902657081373e-05,"hidden-states-l1/_layer_07_moe::shared_experts":0.03204141939180129,"hidden-states-absmax/_layer_46_gated_delta::out_proj":160,"hidden-states-rms/_layer_53_gated_delta::key":0.1827735748278041,"hidden-states-l1/_layer_42_gated_delta::gate":0.9532326404133606,"moe/layer_27/act_rms_max":0.20027457177639008,"grad-rms/layer_27/moe::shared_experts":1.3680950311924357e-06,"hidden-states-absmax/_layer_24_mlp":48.5,"param-mean/_layer_53_gated_delta::A_log":1.246252179145813,"hidden-states-mean/_layer_18_residual_mlp":-0.0013304943407916236,"update-absmax/_layer_12_gated_delta::out_proj":0.00390625,"grad-rms/layer_17/gated_delta::out_proj":5.163779907138469e-07,"update-l1/_layer_00_embedding":0.001145453774370253,"grad-rms/layer_43/gated_delta::out_proj":3.869803060450068e-07,"hidden-states-mean/_layer_27_gated_delta::out_proj":-0.0008226807511440815,"learning-rate":0.0005000000237487257,"param-mean/_layer_19_gated_delta::A_log":1.059867024421692,"grad-rms/layer_35/moe::expert_fc1":2.0313774240610756e-08,"update-rms/_layer_06_gated_delta::out_norm":0.0008523095729839441,"update-mean/_layer_46_gated_delta::out_proj":-2.179438212124296e-07,"hidden-states-mean/_layer_52_gated_delta::query":0.001255460233147172,"hidden-states-absmax/_layer_19_gated_delta::g":18.75,"grad-rms/layer_23/gated_delta::A_log":4.732644115501504e-06,"param-absmax/_layer_12_gated_delta::A_log":1.8816908597946167,"hidden-states-absmax/_layer_15_gated_delta::out_proj":11.3125,"hidden-states-rms/_layer_26_pre_mlp_layernorm":0.16849261926288456,"hidden-states-mean/_layer_31_gated_delta::query":0.05785811637140707,"grad-rms/layer_52/moe::expert_fc1":1.8927985893914495e-08,"param-absmax/_layer_52_gated_delta::A_log":2.1580970287323,"hidden-states-absmax/_layer_03_residual_attn":26.5,"grad-rms/layer_14/moe::shared_expert_fc2":2.002023156147172e-07,"param-mean/_layer_40_gated_delta::out_proj":6.250800652196631e-05,"param-mean/_layer_07_gated_delta::conv1d":-0.0005613104440271854,"hidden-states-l1/_layer_44_residual_mlp":3.0281828675488076,"hidden-states-l1/_layer_46_gated_delta::beta":2.0061981685397825,"update-rms/_layer_52_gated_delta::A_log":0.0007181093195292904,"grad-rms/layer_31/gated_delta::dt_bias":1.3353671071443134e-06,"hidden-states-rms/_layer_48_pre_mlp_layernorm":0.2633981641809782,"hidden-states-l1/_layer_53_gated_delta::core_attention":0.0033101399184185167,"grad-rms/layer_06/moe::shared_expert_fc2":1.3627431658438217e-07,"grad-rms/layer_54/gated_delta::out_norm":1.61717344803687e-05,"update-rms/_layer_44_gated_delta::A_log":0.000698261593192311,"update-mean/_layer_01_gated_delta::A_log":-0.0002000780077651143,"param-absmax/_layer_37_gated_delta::dt_bias":0.8589736223220825,"update-rms/_layer_50_attention::o_proj":0.00027177371636970195,"grad-absmax/layer_30/gated_delta::conv1d":0.00018285249825567007,"hidden-states-mean/_layer_46_gated_delta::out_norm":-0.005674306979692814,"param-l1/attention::o_proj":0.03539538835840566,"hidden-states-l1/_layer_17_mlp":0.10213257099778307,"hidden-states-l1/_layer_39_moe::shared_experts":0.11510477097091329,"param-rms/_layer_07_gated_delta::A_log":1.3029710820546925,"grad-absmax/layer_51/gated_delta::dt_bias":3.429703792789951e-05,"update-l1/gated_delta::A_log":0.0005079386939950363,"hidden-states-rms/_layer_42_gated_delta::out_proj":0.5371832544810164,"hidden-states-absmax/_layer_48_gated_delta::in_proj":34.5,"hidden-states-l1/_layer_33_gated_delta::query":0.14369739814965704,"param-rms/_layer_28_gated_delta::A_log":1.3141257798134425,"hidden-states-rms/_layer_14_mlp":0.12593948214121328,"hidden-states-l1/_layer_46_moe::shared_experts":0.3446170204421306,"hidden-states-absmax/_layer_17_residual_attn":494,"grad-absmax/layer_10/moe::expert_fc1":1.26972236103029e-06,"param-mean/_layer_24_attention::o_proj":-2.1218490928731626e-06,"moe/layer_44/act_rms_median":0.22238703072071075,"hidden-states-absmax/_layer_22_residual_mlp":500,"hidden-states-mean/_layer_20_moe::shared_experts":-0.0015504976720211116,"hidden-states-mean/_layer_01_gated_delta::key":0.037624982014296295,"hidden-states-mean/_layer_11_gated_delta::out_proj":-0.0006527677374466047,"hidden-states-l1/_layer_55_gated_delta::gate":1.774474832352976,"grad-rms/layer_17/moe::shared_expert_fc1":3.375299694590885e-07,"param-l1/_layer_14_gated_delta::out_proj":0.03437976911664009,"hidden-states-rms/_layer_56_mlp":3.5257239597758674,"hidden-states-rms/_layer_52_gated_delta::g":8.455652726290728,"grad-rms/layer_09/moe::shared_experts":6.616178636260748e-07,"hidden-states-l1/_layer_17_input_layernorm":0.1589122455015475,"update-mean/_layer_48_gated_delta::out_proj":-5.82083998779126e-08,"grad-absmax/layer_15/moe::shared_expert_fc1":1.931262704601977e-05,"hidden-states-rms/_layer_17_gated_delta::gate":1.1531948931562048,"hidden-states-l1/_layer_50_pre_mlp_layernorm":0.18193729900566327,"update-mean/_layer_11_gated_delta::A_log":-9.192293509840965e-05,"param-l1/_layer_42_gated_delta::out_proj":0.03469468653202057,"hidden-states-absmax/_layer_19_pre_mlp_layernorm":5.59375,"hidden-states-rms/_layer_25_mlp":0.3075159725379082,"moe/layer_10/act_rms_max":0.07462135702371597,"param-absmax/_layer_10_gated_delta::A_log":1.7872110605239868,"param-absmax/_layer_32_gated_delta::dt_bias":0.4873639941215515,"update-l1/_layer_36_gated_delta::A_log":0.0004259888082742691,"hidden-states-mean/_layer_42_moe::shared_experts":0.00010494570392933684,"update-l1/_layer_51_gated_delta::conv1d":0.0006096640136092901,"hidden-states-rms/_layer_44_gated_delta::query":0.20483663366427698,"hidden-states-rms/_layer_50_attention::k_layernorm":3.915457887284882,"hidden-states-l1/_layer_23_pre_mlp_layernorm":0.10670421054413444,"hidden-states-rms/_layer_23_input_layernorm":0.36779052002179463,"param-absmax/_layer_06_gated_delta::out_proj":0.921875,"param-mean/_layer_39_gated_delta::conv1d":0.00030973737011663616,"grad-absmax/layer_44/moe::shared_expert_fc2":1.9281793356640264e-05,"hidden-states-l1/_layer_28_gated_delta::beta":2.5759814472176368,"hidden-states-rms/_layer_27_gated_delta::g":7.748506433168017,"grad-rms/gated_delta::dt_bias":2.127299166596089e-05,"param-mean/_layer_45_gated_delta::A_log":0.9166349768638611,"grad-rms/layer_40/moe::shared_expert_fc1":1.922023639956661e-07,"hidden-states-mean/_layer_38_gated_delta::in_proj":-0.17568151927821024,"update-mean/_layer_52_gated_delta::A_log":0.00019780034199357033,"hidden-states-mean/_layer_03_gated_delta::key":0.010265644932168118,"param-rms/_layer_02_mlp::linear_fc1":0.046596194659189814,"update-absmax/_layer_33_gated_delta::out_norm":0.001953125,"hidden-states-absmax/_layer_08_gated_delta::beta":7.28125,"grad-absmax/moe::shared_experts":0.00011000368249369785,"hidden-states-rms/_layer_29_gated_delta::in_proj":1.7332777716556944,"update-mean/_layer_33_gated_delta::out_norm":-6.103515625e-05,"hidden-states-l1/_layer_43_moe::shared_experts":0.20636498407900145,"hidden-states-absmax/gated_delta::value":19,"grad-rms/layer_24/attention::linear_qkv":2.1058242313368033e-06,"hidden-states-rms/_layer_32_pre_mlp_layernorm":0.19054288347937637,"hidden-states-mean/_layer_08_input_layernorm":-0.0009203227169823906,"hidden-states-l1/_layer_31_residual_mlp":1.754752129882828,"param-rms/_layer_05_gated_delta::out_proj":0.046051530980595196,"hidden-states-absmax/_layer_56_input_layernorm":35.5,"hidden-states-rms/_layer_29_mlp":1.0446164886700056,"param-rms/_layer_01_gated_delta::A_log":1.4790524272950496,"hidden-states-mean/_layer_02_pre_mlp_layernorm":0.0010432496280744228,"hidden-states-rms/_layer_07_gated_delta::in_proj":1.146293175447849,"grad-rms/layer_43/moe::shared_experts":2.372510213676098e-06,"hidden-states-rms/_layer_26_residual_attn":3.05186189888631,"update-l1/_layer_25_gated_delta::out_norm":0.000457763671875,"hidden-states-mean/_layer_52_residual_mlp":0.12556545443123035,"update-l1/_layer_53_gated_delta::conv1d":0.000600624131038785,"update-absmax/_layer_08_gated_delta::A_log":0.0023483633995056152,"param-mean/_layer_01_gated_delta::out_norm":0.3398590087890625,"update-absmax/_layer_19_gated_delta::dt_bias":0.0011035054922103882,"hidden-states-l1/_layer_37_gated_delta::key":0.05482392294096028,"hidden-states-mean/_layer_03_gated_delta::out_proj":0.00044600976476375935,"moe/layer_42/act_rms_min":0.07914148271083832,"param-mean/_layer_21_attention::linear_qkv":0.00017868448048830032,"param-mean/_layer_11_gated_delta::out_proj":6.945242603251245e-06,"param-rms/_layer_08_gated_delta::out_norm":0.34867846833802457,"param-absmax/_layer_08_gated_delta::dt_bias":0.5597469806671143,"hidden-states-l1/_layer_02_pre_mlp_layernorm":0.04487230418590357,"hidden-states-mean/_layer_12_gated_delta::g":-5.611417008807303,"hidden-states-rms/_layer_17_gated_delta::g":7.0446723965934375,"param-absmax/_layer_34_gated_delta::dt_bias":0.6246829032897949,"hidden-states-mean/_layer_11_mlp":-0.00012139652964077498,"hidden-states-rms/_layer_04_gated_delta::out_norm":0.03117049242189754,"hidden-states-absmax/_layer_24_attention::o_proj":35.25,"hidden-states-mean/gated_delta::g":-5.293437878400589,"hidden-states-mean/_layer_41_pre_mlp_layernorm":0.0003114489744095772,"grad-rms/layer_53/moe::expert_fc1":1.8074226876191146e-08,"grad-rms/layer_37/moe::shared_experts":1.6479270710409422e-06,"update-mean/_layer_42_gated_delta::out_norm":6.103515625e-05,"hidden-states-absmax/_layer_32_pre_mlp_layernorm":4.09375,"moe/layer_45/act_rms_max":0.5244803428649902,"hidden-states-rms/_layer_27_input_layernorm":0.3241471783926706,"grad-absmax/layer_30/moe::expert_fc2":2.3839506866352167e-06,"moe/layer_21/act_rms_max":0.154743954539299,"param-l1/gated_delta::out_norm":0.7485978262765067,"hidden-states-absmax/_layer_18_gated_delta::value":1.390625,"hidden-states-l1/_layer_34_gated_delta::value":0.0582060100979817,"hidden-states-absmax/_layer_53_gated_delta::out_norm":225,"hidden-states-l1/_layer_14_gated_delta::beta":2.5478780922744386,"hidden-states-l1/_layer_08_moe::routed_experts":0.01758799544519911,"hidden-states-rms/_layer_09_moe::router":0.07216878508256834,"hidden-states-l1/_layer_12_gated_delta::out_norm":0.04113623245194345,"param-mean/_layer_51_gated_delta::conv1d":-0.0005644956254400313,"hidden-states-rms/_layer_35_pre_mlp_layernorm":0.20734332851181825,"update-l1/_layer_10_gated_delta::out_proj":0.00015165662625804543,"grad-absmax/layer_35/gated_delta::dt_bias":8.61251464812085e-06,"moe/layer_01/act_rms_max":0,"update-l1/_layer_21_attention::o_proj":0.00011641241871984676,"hidden-states-mean/_layer_39_gated_delta::out_proj":0.0002148409804791636,"param-absmax/_layer_36_gated_delta::out_norm":1.6484375,"hidden-states-mean/_layer_20_residual_attn":-0.0008373313883118497,"grad-rms/layer_49/moe::shared_experts":3.519465791449715e-06,"grad-absmax/layer_39/gated_delta::out_proj":0.0001353388506686315,"hidden-states-l1/_layer_43_residual_mlp":2.846552234477071,"hidden-states-rms/_layer_53_gated_delta::out_proj":1.5576804188821303,"param-l1/_layer_49_attention::o_proj":0.04360345005989075,"grad-rms/layer_49/attention::linear_qkv":9.298978681038237e-07,"param-mean/_layer_12_gated_delta::dt_bias":0.36468666791915894,"hidden-states-l1/_layer_46_residual_mlp":3.491378008291484,"grad-absmax/layer_54/gated_delta::out_norm":5.676315049640834e-05,"param-absmax/_layer_18_gated_delta::out_proj":1.1171875,"param-l1/_layer_48_gated_delta::out_proj":0.031983379274606705,"hidden-states-l1/_layer_12_gated_delta::gate":0.9432167939206116,"grad-absmax/layer_22/moe::expert_fc2":1.8586933947517537e-06,"hidden-states-l1/_layer_39_mlp":0.23895722495999633,"hidden-states-mean/_layer_17_gated_delta::out_norm":-0.000658779250759393,"grad-absmax/layer_40/gated_delta::A_log":8.13405404187506e-06,"param-mean/_layer_34_gated_delta::out_norm":0.60272216796875,"hidden-states-l1/_layer_40_gated_delta::beta":2.060445315890946,"update-mean/_layer_51_gated_delta::out_norm":0.000244140625,"hidden-states-l1/_layer_37_residual_mlp":2.3187754534215275,"hidden-states-rms/_layer_18_moe::shared_experts":0.08545044998713919,"hidden-states-absmax/_layer_54_gated_delta::in_proj":38.25,"grad-absmax/layer_40/gated_delta::out_norm":3.5639448469737545e-05,"hidden-states-mean/_layer_39_gated_delta::g":-3.6356632275179894,"hidden-states-rms/_layer_18_gated_delta::g":5.2197744682666745,"grad-absmax/layer_43/moe::shared_expert_fc1":3.610472413129173e-05,"update-mean/_layer_37_gated_delta::A_log":4.684313898906112e-05,"hidden-states-absmax/_layer_29_gated_delta::g":19.875,"hidden-states-mean/_layer_51_gated_delta::core_attention":0.002315726872918233,"update-rms/_layer_00_embedding":0.0017723883868589238,"grad-absmax/layer_05/moe::shared_expert_fc2":2.2056526631786255e-06,"hidden-states-rms/_layer_48_mlp":0.8506192030105149,"grad-rms/layer_32/gated_delta::out_proj":3.1715975855516015e-07,"hidden-states-rms/_layer_27_gated_delta::out_norm":0.1524186713465953,"hidden-states-absmax/_layer_01_gated_delta::out_norm":21.125,"hidden-states-l1/_layer_08_residual_mlp":0.28359440369539307,"update-rms/_layer_46_gated_delta::out_norm":0.0025371838001497227,"update-rms/_layer_09_gated_delta::out_norm":0.0010935347208965403,"param-l1/_layer_14_gated_delta::conv1d":0.04125472530722618,"hidden-states-l1/_layer_20_gated_delta::g":5.16913284887467,"hidden-states-l1/_layer_52_gated_delta::core_attention":0.009998494088470231,"hidden-states-absmax/_layer_16_gated_delta::gate":13.0625,"lm-global-supervised-tokens":1.2934637e+07,"hidden-states-l1/_layer_09_gated_delta::query":0.04811101235282569,"grad-absmax/layer_48/gated_delta::out_norm":6.372302595991641e-05,"grad-absmax/layer_17/moe::expert_fc2":9.124211146627204e-07,"param-absmax/_layer_23_gated_delta::dt_bias":0.6223967671394348,"hidden-states-mean/_layer_24_attention::core_attention":-0.0008737965713683154,"param-l1/_layer_04_gated_delta::conv1d":0.03291129693388939,"hidden-states-absmax/_layer_51_residual_attn":740,"param-mean/lm_head":0.00013984426914248616,"hidden-states-mean/_layer_43_gated_delta::key":0.028101877865419133,"param-absmax/_layer_44_gated_delta::out_norm":1.671875,"param-absmax/_layer_26_gated_delta::A_log":1.893563985824585,"hidden-states-mean/_layer_25_gated_delta::in_proj":-0.0174047610110776,"param-l1/_layer_55_gated_delta::out_proj":0.03130744397640228,"param-absmax/_layer_51_gated_delta::conv1d":0.52734375,"grad-rms/layer_43/gated_delta::out_norm":1.1548204045403405e-05,"hidden-states-rms/_layer_11_gated_delta::core_attention":0.0027181769960184384,"grad-rms/layer_17/gated_delta::out_norm":1.3399089842301457e-05,"update-l1/_layer_51_gated_delta::out_norm":0.00048828125,"hidden-states-rms/_layer_45_gated_delta::g":7.693291902785106,"grad-absmax/layer_50/moe::expert_fc1":1.2381788110360503e-05,"param-rms/_layer_17_gated_delta::out_norm":0.6589608062165476,"hidden-states-l1/_layer_53_gated_delta::key":0.10390498237669257,"hidden-states-l1/_layer_18_gated_delta::gate":0.7720873680218897,"hidden-states-l1/_layer_31_gated_delta::query":0.1462898726808873,"hidden-states-l1/_layer_54_moe::routed_experts":0.07818683946965901,"update-absmax/_layer_43_gated_delta::out_norm":0.0078125,"hidden-states-absmax/_layer_56_gated_delta::core_attention":0.4375,"param-l1/_layer_32_gated_delta::out_proj":0.034787461161613464,"moe/layer_24/act_rms_median":0.09360723197460175,"hidden-states-rms/_layer_33_gated_delta::beta":1.9000472041019532,"param-mean/_layer_25_gated_delta::out_norm":0.681243896484375,"param-mean/_layer_06_gated_delta::A_log":1.2835592031478882,"grad-absmax/layer_31/gated_delta::dt_bias":3.663549250632059e-06,"hidden-states-l1/_layer_31_gated_delta::key":0.13434712700213516,"hidden-states-absmax/_layer_26_moe::routed_experts":11.3125,"grad-absmax/layer_44/gated_delta::out_norm":3.8178262911969796e-05,"hidden-states-rms/_layer_53_gated_delta::out_norm":0.6385783703015674,"update-rms/_layer_11_gated_delta::A_log":0.0006250937085095758,"grad-rms/layer_10/moe::expert_fc2":2.06735371928227e-08,"hidden-states-rms/_layer_27_gated_delta::query":0.14514751341012475,"hidden-states-l1/_layer_47_residual_attn":3.5812084175596737,"grad-rms/layer_07/gated_delta::A_log":1.472210419045317e-06,"param-l1/_layer_43_gated_delta::dt_bias":0.3516674041748047,"param-absmax/attention::linear_qkv":0.8125,"hidden-states-l1/_layer_52_moe::router":0.010327688507666351,"hidden-states-l1/_layer_44_gated_delta::in_proj":1.2591050945428066,"hidden-states-l1/gated_delta::out_proj":0.35197318907096986,"grad-rms/layer_54/gated_delta::out_proj":5.486319201929818e-07,"param-rms/_layer_43_gated_delta::A_log":1.1954105810677753,"update-mean/_layer_26_gated_delta::dt_bias":4.05532045988366e-05,"param-rms/_layer_57_lm_head":0.05594011503772113,"update-l1/_layer_20_gated_delta::out_norm":0.000701904296875,"hidden-states-l1/_layer_25_moe::router":0.010255072654099886,"update-mean/_layer_56_gated_delta::conv1d":1.1008661203959491e-05,"hidden-states-l1/_layer_48_residual_mlp":3.9796273684057435,"param-absmax/_layer_51_gated_delta::out_proj":2.15625,"param-absmax/_layer_32_gated_delta::conv1d":0.66015625,"hidden-states-absmax/_layer_04_moe::routed_experts":4.25,"hidden-states-rms/_layer_20_gated_delta::gate":1.379376724109002,"hidden-states-rms/_layer_15_moe::shared_experts":0.07364979931453992,"hidden-states-l1/_layer_19_gated_delta::out_proj":0.1354651612597896,"grad-rms/layer_30/moe::shared_experts":2.1340213784020197e-06,"hidden-states-l1/_layer_17_moe::shared_experts":0.060312892980920435,"param-mean/_layer_23_gated_delta::dt_bias":0.024221373721957207,"update-rms/gated_delta::out_norm":0.0015520245840896772,"hidden-states-rms/_layer_27_gated_delta::out_proj":0.32420740546368343,"update-absmax/_layer_02_gated_delta::dt_bias":0.0017014145851135254,"param-absmax/_layer_19_gated_delta::out_proj":1.5078125,"grad-rms/layer_21/moe::shared_expert_fc1":5.03211089727491e-07,"hidden-states-rms/_layer_05_moe::router":0.07216878508256834,"update-mean/_layer_32_gated_delta::A_log":-2.1000392735004425e-05,"hidden-states-rms/_layer_11_residual_mlp":0.9233563597340934,"grad-absmax/layer_30/moe::shared_expert_fc2":2.5519449991406873e-05,"hidden-states-mean/_layer_36_input_layernorm":-0.0016238429423904945,"update-absmax/_layer_34_gated_delta::out_norm":0.00390625,"hidden-states-l1/_layer_19_gated_delta::core_attention":0.0004866723318528929,"hidden-states-mean/_layer_02_gated_delta::value":0.003784049986264108,"param-absmax/_layer_46_gated_delta::conv1d":0.60546875,"hidden-states-rms/_layer_18_moe::router":0.07216878508256834,"grad-absmax/layer_48/moe::shared_experts":5.6394721468677744e-05,"param-rms/gated_delta::out_proj":0.04399435764914943,"param-absmax/_layer_09_gated_delta::out_proj":1.046875,"hidden-states-mean/_layer_44_gated_delta::beta":0.6853565794654584,"update-mean/gated_delta::conv1d":-1.3895534456590825e-06,"moe/layer_12/act_rms_max":0.07466447353363037,"grad-rms/layer_49/moe::shared_expert_fc2":2.024311032790717e-07,"hidden-states-absmax/_layer_37_gated_delta::in_proj":21,"hidden-states-rms/_layer_53_gated_delta::beta":2.66885069008937,"param-absmax/_layer_04_gated_delta::conv1d":0.45703125,"param-l1/_layer_11_gated_delta::conv1d":0.03594944253563881,"update-mean/_layer_38_gated_delta::out_norm":-0.000244140625,"hidden-states-mean/_layer_42_pre_mlp_layernorm":0.0006075810666851481,"update-absmax/_layer_52_gated_delta::conv1d":0.00390625,"param-absmax/_layer_31_gated_delta::out_norm":2.34375,"hidden-states-rms/_layer_46_moe::router":0.07216878508256834,"grad-absmax/layer_07/moe::shared_experts":4.231186721881386e-06,"hidden-states-l1/_layer_27_residual_attn":1.298053300148642,"grad-rms/layer_32/moe::shared_expert_fc1":2.1929893379336116e-07,"moe/layer_38/act_rms_min":0.08504024893045425,"update-rms/_layer_33_gated_delta::out_norm":0.0008457279333832408,"hidden-states-rms/_layer_05_pre_mlp_layernorm":0.0624087606373965,"update-rms/_layer_44_gated_delta::out_proj":0.00019973886432938042,"hidden-states-l1/_layer_55_pre_mlp_layernorm":0.10133409218002465,"grad-absmax/layer_42/gated_delta::conv1d":0.0007457403698936105,"hidden-states-l1/_layer_33_gated_delta::g":5.781937334768455,"param-mean/_layer_56_gated_delta::dt_bias":0.8782429099082947,"update-l1/_layer_27_gated_delta::out_norm":0.001129150390625,"grad-rms/layer_30/moe::router":7.046481210075606e-07,"hidden-states-mean/_layer_28_moe::router":0.010261876277029717,"hidden-states-absmax/_layer_41_gated_delta::in_proj":36,"grad-rms/layer_44/gated_delta::conv1d":7.35309562362487e-05,"hidden-states-l1/_layer_27_input_layernorm":0.17018600346875232,"update-l1/_layer_17_gated_delta::dt_bias":0.0004596817889250815,"hidden-states-absmax/_layer_26_gated_delta::g":15.1875,"hidden-states-l1/_layer_07_input_layernorm":0.18942483537493174,"hidden-states-rms/_layer_29_moe::routed_experts":0.13051982500823003,"hidden-states-l1/_layer_18_gated_delta::out_proj":0.11949889296280061,"grad-rms/layer_53/moe::shared_expert_fc1":1.0933530189108444e-06,"update-rms/_layer_47_attention::linear_qkv":0.00018745606768034485,"hidden-states-absmax/_layer_54_residual_mlp":2800,"param-l1/_layer_51_gated_delta::A_log":1.2166587114334106,"hidden-states-mean/_layer_06_gated_delta::query":0.01783554841107707,"hidden-states-rms/_layer_55_input_layernorm":0.8653688863636749,"param-absmax/_layer_04_gated_delta::dt_bias":0.6555817723274231,"grad-rms/layer_19/moe::expert_fc1":2.0350982670228675e-08,"hidden-states-rms/_layer_16_gated_delta::out_norm":0.07150637578935813,"hidden-states-l1/_layer_39_gated_delta::beta":1.1554409450818235,"hidden-states-absmax/_layer_14_residual_mlp":494,"hidden-states-rms/_layer_30_gated_delta::query":0.34867247396123113,"hidden-states-l1/_layer_27_moe::router":0.010284673815516442,"param-l1/_layer_37_gated_delta::out_proj":0.035253819078207016,"update-rms/_layer_05_gated_delta::dt_bias":0.0005194459402604843,"hidden-states-l1/_layer_06_moe::router":0.01027064126561344,"hidden-states-rms/_layer_09_input_layernorm":0.3279103726168514,"param-mean/_layer_34_gated_delta::dt_bias":0.2859063148498535,"grad-absmax/layer_02/gated_delta::out_norm":0.00015281993546523154,"hidden-states-absmax/_layer_53_gated_delta::value":9.25,"hidden-states-mean/_layer_07_gated_delta::g":-4.16419889500128,"grad-rms/layer_20/moe::expert_fc2":2.0197691527046664e-08,"hidden-states-l1/_layer_52_gated_delta::gate":1.5517074527391734,"grad-rms/layer_09/moe::expert_fc1":1.5170705684896335e-08,"hidden-states-absmax/_layer_37_input_layernorm":26.5,"hidden-states-absmax/_layer_48_residual_mlp":616,"hidden-states-absmax/_layer_33_residual_attn":502,"update-rms/_layer_23_gated_delta::dt_bias":0.0006835726144715855,"update-absmax/gated_delta::out_proj":0.0078125,"hidden-states-mean/_layer_20_gated_delta::in_proj":-0.0005170534633433696,"update-mean/_layer_04_gated_delta::conv1d":-5.12845463163103e-06,"param-rms/_layer_35_gated_delta::dt_bias":0.30413937783738854,"grad-rms/layer_18/gated_delta::conv1d":1.8528991277546e-05,"hidden-states-mean/_layer_54_moe::shared_experts":0.0009823184234432486,"update-rms/_layer_18_gated_delta::out_norm":0.001708984375,"hidden-states-mean/_layer_07_moe::router":0.01024086191111595,"grad-rms/layer_48/gated_delta::dt_bias":2.775613280222176e-05,"hidden-states-absmax/_layer_11_gated_delta::core_attention":0.146484375,"hidden-states-rms/_layer_17_gated_delta::out_norm":0.07395407073720053,"hidden-states-rms/_layer_20_gated_delta::key":0.14756992173985317,"hidden-states-mean/_layer_46_moe::routed_experts":-0.00015513469969168629,"hidden-states-absmax/_layer_34_residual_attn":502,"hidden-states-absmax/_layer_26_input_layernorm":18.25,"hidden-states-l1/_layer_25_moe::routed_experts":0.06673736025554898,"hidden-states-absmax/_layer_12_pre_mlp_layernorm":5.96875,"hidden-states-absmax/_layer_27_gated_delta::value":2.328125,"hidden-states-absmax/_layer_50_moe::router":0.9978412389755249,"grad-rms/layer_38/gated_delta::A_log":5.686601951264768e-06,"hidden-states-l1/_layer_03_gated_delta::gate":0.691921793451523,"grad-rms/layer_39/moe::shared_experts":2.170839205455966e-06,"hidden-states-absmax/_layer_10_input_layernorm":12.5625,"hidden-states-l1/_layer_37_moe::routed_experts":0.073909863998831,"param-l1/_layer_01_mlp::linear_fc1":0.0349157489836216,"grad-absmax/layer_41/moe::router":3.0936134862713516e-05,"update-mean/_layer_56_gated_delta::out_norm":0.0001220703125,"update-mean/_layer_09_gated_delta::A_log":-0.00035295402631163597,"hidden-states-absmax/_layer_44_residual_attn":502,"grad-absmax/layer_29/moe::expert_fc2":1.3686352531294688e-06,"grad-absmax/layer_23/moe::router":6.403510633390397e-05,"hidden-states-absmax/_layer_36_gated_delta::out_proj":53.5,"update-absmax/_layer_55_gated_delta::A_log":0.0015022754669189453,"hidden-states-mean/_layer_41_residual_mlp":-0.09010436714939679,"hidden-states-rms/_layer_42_moe::routed_experts":0.20983976484965297,"hidden-states-mean/_layer_41_gated_delta::out_norm":-0.0013135237788139318,"hidden-states-l1/_layer_22_moe::router":0.010251051202583504,"param-l1/_layer_08_gated_delta::dt_bias":0.2231394350528717,"hidden-states-mean/_layer_34_gated_delta::query":0.04288658638436632,"grad-rms/layer_31/moe::expert_fc2":2.0907696311221964e-08,"hidden-states-mean/_layer_35_moe::router":0.010168227431095033,"update-rms/_layer_57_lm_head":0.0002740292023238406,"hidden-states-absmax/_layer_25_mlp":29.25,"hidden-states-rms/_layer_39_residual_attn":7.4566751306305195,"param-mean/_layer_14_gated_delta::out_proj":1.6856802176334895e-05,"grad-rms/layer_54/gated_delta::A_log":4.0701379410338945e-06,"grad-absmax/layer_14/moe::expert_fc2":1.77352364971739e-06,"moe/layer_19/act_rms_median":0.05345658212900162,"hidden-states-absmax/_layer_50_residual_attn":616,"param-l1/_layer_53_gated_delta::dt_bias":0.6845307946205139,"hidden-states-mean/_layer_23_moe::routed_experts":-7.641007790635278e-05,"hidden-states-l1/_layer_43_gated_delta::key":0.12260945857876046,"update-rms/_layer_41_gated_delta::out_norm":0.0012918707573557572,"param-absmax/_layer_10_gated_delta::dt_bias":0.6859127283096313,"hidden-states-rms/_layer_28_gated_delta::g":8.3926533954602,"grad-rms/layer_48/moe::expert_fc2":2.110206873576336e-08,"hidden-states-mean/_layer_55_moe::shared_experts":0.0004407913018216255,"moe/layer_23/act_rms_min_to_median":0.3547845933588511,"update-rms/_layer_04_gated_delta::A_log":0.0004007575725968097,"hidden-states-mean/_layer_21_attention::k_layernorm":-0.0794934576145956,"hidden-states-mean/_layer_34_residual_attn":-0.048562271721633875,"hidden-states-mean/_layer_35_gated_delta::key":-0.018833371261760146,"update-rms/_layer_55_gated_delta::out_proj":0.00018357790323960348,"param-absmax/_layer_18_gated_delta::conv1d":0.8515625,"hidden-states-rms/_layer_15_residual_attn":1.2257051570633137,"hidden-states-mean/_layer_11_gated_delta::key":0.012738760216684367,"hidden-states-l1/_layer_10_gated_delta::key":0.062395423638275105,"update-absmax/attention::linear_qkv":0.00390625,"grad-absmax/layer_08/moe::router":6.525858043460175e-05,"param-mean/_layer_26_gated_delta::dt_bias":0.03458835184574127,"param-rms/_layer_02_mlp::linear_fc2":0.046158137044640196,"hidden-states-rms/_layer_49_attention::o_proj":0.9992718428848393,"param-l1/_layer_46_gated_delta::dt_bias":0.5893617272377014,"hidden-states-absmax/_layer_51_gated_delta::beta":10.8125,"hidden-states-l1/_layer_22_attention::k_layernorm":2.6858735902158726,"update-absmax/_layer_38_gated_delta::dt_bias":0.0013551115989685059,"update-absmax/_layer_55_gated_delta::conv1d":0.0030059814453125,"grad-rms/layer_47/moe::expert_fc1":2.0069020135879608e-08,"grad-rms/layer_08/gated_delta::conv1d":7.267871113145423e-06,"update-mean/_layer_24_attention::o_proj":-3.8867028706590645e-07,"hidden-states-l1/_layer_16_gated_delta::out_proj":0.10206089640389639,"update-l1/_layer_54_gated_delta::A_log":0.0004409300163388252,"update-mean/_layer_03_gated_delta::conv1d":9.970198107112083e-07,"hidden-states-l1/_layer_56_gated_delta::g":4.200103783619855,"hidden-states-absmax/_layer_37_mlp":26.625,"hidden-states-mean/_layer_54_moe::router":0.010351421715048426,"hidden-states-absmax/_layer_27_gated_delta::beta":8.6875,"grad-rms/layer_11/moe::expert_fc1":1.7824455539515506e-08,"grad-absmax/layer_01/gated_delta::conv1d":0.00038386511732824147,"hidden-states-mean/_layer_27_moe::routed_experts":-1.7037984270385512e-05,"grad-rms/layer_33/gated_delta::conv1d":6.709064766625286e-06,"grad-rms/layer_52/gated_delta::dt_bias":1.3178495453576811e-05,"hidden-states-l1/_layer_45_residual_attn":3.092771898838508,"hidden-states-l1/_layer_02_gated_delta::core_attention":0.0026856820041009193,"param-rms/_layer_04_gated_delta::out_norm":0.20292359233730775,"hidden-states-absmax/_layer_02_mlp":8,"update-rms/_layer_12_gated_delta::A_log":0.0005221177178250594,"hidden-states-l1/_layer_03_gated_delta::out_norm":0.01345378600255084,"hidden-states-l1/_layer_24_attention::o_proj":0.3853363556070752,"hidden-states-mean/_layer_05_gated_delta::beta":-0.22870889052258617,"hidden-states-l1/_layer_56_moe::routed_experts":0.04634076910331079,"moe/layer_23/act_rms_min":0.03665519878268242,"param-rms/_layer_13_attention::linear_qkv":0.04137255274755754,"hidden-states-mean/_layer_11_gated_delta::g":-6.380377099558961,"hidden-states-absmax/_layer_46_moe::shared_experts":125.5,"update-rms/_layer_35_gated_delta::A_log":0.0005987306238078516,"param-absmax/_layer_30_gated_delta::out_norm":1.9140625,"hidden-states-rms/_layer_41_input_layernorm":0.5750554359383542,"update-mean/_layer_28_gated_delta::A_log":-2.631591632962227e-05,"param-absmax/_layer_27_gated_delta::conv1d":0.80859375,"update-mean/_layer_34_gated_delta::conv1d":-4.5076353671902325e-06,"hidden-states-mean/_layer_48_gated_delta::out_norm":-0.006316044189458392,"hidden-states-mean/_layer_07_gated_delta::query":0.014393913412858655,"hidden-states-mean/_layer_25_gated_delta::value":0.0038279576599398444,"update-l1/_layer_46_gated_delta::out_proj":0.00013446994125843048,"hidden-states-l1/_layer_39_gated_delta::value":0.03702486002741576,"update-absmax/_layer_16_gated_delta::dt_bias":0.0012875348329544067,"grad-rms/layer_43/gated_delta::conv1d":2.038186634723816e-05,"param-absmax/_layer_00_embedding":3.15625,"param-l1/_layer_48_gated_delta::A_log":1.3156193494796753,"hidden-states-absmax/_layer_20_pre_mlp_layernorm":5.875,"hidden-states-mean/_layer_49_residual_mlp":-0.10708328122893349,"hidden-states-mean/_layer_48_input_layernorm":-0.0009531209333794932,"hidden-states-rms/_layer_03_residual_attn":0.5358655961574933,"grad-absmax/layer_02/gated_delta::dt_bias":8.561481081414968e-06,"update-rms/_layer_12_gated_delta::conv1d":0.0007653243445158466,"hidden-states-rms/_layer_34_gated_delta::query":0.5276153179846919,"update-mean/_layer_35_gated_delta::out_proj":-9.61727408821389e-09,"hidden-states-l1/_layer_42_gated_delta::value":0.028235476104673876,"hidden-states-mean/_layer_29_pre_mlp_layernorm":-0.000332429579040954,"hidden-states-rms/_layer_44_gated_delta::g":4.211803478014789,"update-mean/_layer_15_gated_delta::dt_bias":0.00016764854080975056,"grad-rms/layer_11/gated_delta::A_log":2.523540062705626e-06,"hidden-states-rms/_layer_44_gated_delta::core_attention":0.005809268911631662,"grad-absmax/layer_04/moe::expert_fc2":9.017994102578086e-07,"update-mean/_layer_19_gated_delta::A_log":-5.8309175074100494e-05,"hidden-states-mean/_layer_38_gated_delta::out_proj":-0.0007958521385389135,"param-l1/_layer_12_gated_delta::out_norm":0.518310546875,"grad-absmax/layer_26/gated_delta::conv1d":0.0005219070008024573,"hidden-states-l1/_layer_18_gated_delta::query":0.055591627131894486,"hidden-states-mean/_layer_33_gated_delta::value":-0.001189070895501482,"hidden-states-l1/_layer_11_residual_attn":0.3405629206374789,"hidden-states-absmax/_layer_48_moe::router":0.997441828250885,"hidden-states-absmax/_layer_49_attention::o_proj":24.625,"hidden-states-l1/_layer_42_gated_delta::g":4.0028843883138165,"grad-absmax/layer_35/moe::shared_expert_fc1":5.5609671107959e-05,"hidden-states-l1/_layer_11_gated_delta::beta":1.9421079089770208,"grad-absmax/layer_39/gated_delta::dt_bias":6.025295988365542e-06,"moe/global/act_min_to_median_mean":0.3574990999342448,"grad-rms/layer_42/gated_delta::out_proj":3.6755784598203693e-07,"grad-rms/layer_37/gated_delta::out_proj":2.86426521098335e-07,"update-absmax/_layer_43_gated_delta::dt_bias":0.001293480396270752,"hidden-states-rms/_layer_48_gated_delta::beta":2.6293018811572457,"hidden-states-l1/_layer_19_gated_delta::beta":2.2392046736722797,"hidden-states-l1/_layer_29_mlp":0.35020948650950035,"update-absmax/_layer_47_attention::o_proj":0.00390625,"hidden-states-mean/_layer_53_gated_delta::value":0.0053886395783590765,"hidden-states-l1/_layer_48_moe::router":0.010241028307442395,"hidden-states-rms/_layer_16_moe::routed_experts":0.054430673077819285,"hidden-states-rms/_layer_10_gated_delta::key":0.11808159916095967,"update-absmax/_layer_02_gated_delta::out_proj":0.001953125,"hidden-states-rms/_layer_50_pre_mlp_layernorm":0.29256947800715066,"hidden-states-rms/_layer_19_residual_attn":1.3983734779172474,"hidden-states-rms/_layer_14_moe::routed_experts":0.04813221964259903,"param-l1/_layer_40_gated_delta::dt_bias":0.30548059940338135,"hidden-states-rms/_layer_45_moe::routed_experts":0.268606023761388,"hidden-states-l1/_layer_54_mlp":0.2657199222707656,"hidden-states-mean/_layer_07_gated_delta::core_attention":-3.474552845924873e-05,"hidden-states-l1/_layer_41_residual_attn":2.5975406882215366,"hidden-states-mean/_layer_45_residual_mlp":-0.09114714308347023,"update-mean/_layer_29_gated_delta::A_log":7.502385415136814e-06,"hidden-states-l1/_layer_30_gated_delta::out_proj":0.2578731486646114,"hidden-states-mean/_layer_35_gated_delta::out_norm":-0.002502220049765116,"hidden-states-l1/_layer_03_gated_delta::in_proj":0.7496585374675362,"param-l1/_layer_27_gated_delta::out_proj":0.03463972732424736,"hidden-states-l1/_layer_36_moe::shared_experts":0.1403097677600107,"moe/layer_30/act_rms_max":0.25258415937423706,"hidden-states-rms/_layer_13_residual_mlp":1.1207406291920685,"update-l1/_layer_34_gated_delta::out_proj":0.00014272589760366827,"hidden-states-rms/_layer_04_gated_delta::in_proj":1.1333481018698983,"hidden-states-l1/_layer_37_moe::router":0.010114200158190752,"hidden-states-rms/_layer_34_moe::router":0.07216878508256834,"update-rms/_layer_55_gated_delta::A_log":0.0007015501834940164,"hidden-states-mean/_layer_31_gated_delta::g":-5.299357569141098,"hidden-states-rms/_layer_16_gated_delta::out_proj":0.14023291219528317,"hidden-states-absmax/_layer_53_gated_delta::query":7.1875,"hidden-states-mean/_layer_31_moe::routed_experts":-0.0001994651212510989,"hidden-states-l1/_layer_08_gated_delta::value":0.02272719602014971,"hidden-states-rms/_layer_53_gated_delta::core_attention":0.011424635604816215,"hidden-states-absmax/_layer_48_gated_delta::core_attention":0.32421875,"hidden-states-absmax/_layer_27_gated_delta::query":5.71875,"update-absmax/_layer_53_gated_delta::dt_bias":0.0018839836120605469,"hidden-states-mean/_layer_03_pre_mlp_layernorm":0.0005543999219901767,"param-mean/gated_delta::conv1d":-0.00018665080115541207,"hidden-states-l1/_layer_20_residual_attn":0.6632028416636994,"update-mean/_layer_46_gated_delta::A_log":9.762635454535484e-05,"param-l1/_layer_28_gated_delta::A_log":1.2398271560668945,"grad-absmax/layer_26/moe::router":5.369570862967521e-05,"hidden-states-rms/_layer_51_gated_delta::in_proj":3.0668589724511137,"param-l1/_layer_36_gated_delta::out_norm":1.0648193359375,"hidden-states-absmax/_layer_20_gated_delta::core_attention":0.10400390625,"hidden-states-l1/_layer_06_moe::routed_experts":0.014370923428295986,"hidden-states-absmax/_layer_14_gated_delta::g":22.375,"hidden-states-l1/_layer_36_gated_delta::key":0.032327581325481095,"update-mean/_layer_33_gated_delta::conv1d":-1.0668350114428904e-05,"hidden-states-absmax/_layer_40_gated_delta::key":7.28125,"moe/layer_48/act_rms_min":0.09500929713249207,"update-l1/_layer_36_gated_delta::out_norm":0.0001220703125,"param-absmax/_layer_55_gated_delta::out_norm":1.7421875,"grad-absmax/layer_24/moe::expert_fc2":9.130444027505291e-07,"hidden-states-rms/_layer_07_gated_delta::value":0.057543357113266205,"grad-rms/layer_29/moe::shared_expert_fc1":2.7676457085788784e-07,"hidden-states-rms/_layer_17_gated_delta::out_proj":0.13827045004905184,"update-rms/_layer_10_gated_delta::out_norm":0.0011287379155826983,"moe/layer_11/act_rms_min":0.009447908028960228,"grad-absmax/layer_06/moe::shared_expert_fc1":1.8586655642138794e-05,"param-l1/_layer_34_gated_delta::out_norm":0.60272216796875,"grad-rms/layer_18/moe::shared_expert_fc1":3.4253527208839164e-07,"grad-absmax/layer_06/gated_delta::conv1d":0.012934147380292416,"hidden-states-rms/_layer_32_gated_delta::value":0.14556630997998954,"hidden-states-l1/_layer_43_gated_delta::g":4.524759831995944,"hidden-states-l1/_layer_26_gated_delta::out_norm":0.06474658873636588,"param-absmax/_layer_29_gated_delta::dt_bias":0.5869240164756775,"update-rms/_layer_26_gated_delta::conv1d":0.0008341557065690086,"hidden-states-mean/_layer_47_moe::shared_experts":8.639247546273923e-05,"hidden-states-mean/_layer_27_moe::router":0.010284673815516442,"hidden-states-absmax/_layer_36_residual_mlp":502,"grad-rms/layer_41/gated_delta::conv1d":2.7591386172538944e-05,"param-l1/lm_head":0.039115045219659805,"grad-absmax/layer_36/gated_delta::out_norm":0.0003378886030986905,"moe/layer_34/act_rms_min":0.06550630927085876,"moe/layer_31/act_rms_max":0.26888570189476013,"grad-absmax/layer_43/moe::expert_fc1":2.0109996512474027e-06,"hidden-states-mean/_layer_10_gated_delta::beta":-1.1206243181972884,"grad-absmax/layer_33/moe::expert_fc1":1.9784338292083703e-06,"param-l1/_layer_55_gated_delta::dt_bias":0.8285908699035645,"hidden-states-absmax/_layer_53_gated_delta::gate":31.125,"hidden-states-mean/_layer_07_gated_delta::key":0.014097541388576195,"param-absmax/_layer_56_gated_delta::conv1d":0.404296875,"hidden-states-absmax/_layer_26_moe::router":0.999817430973053,"hidden-states-rms/_layer_10_input_layernorm":0.2988930230246728,"param-rms/_layer_44_gated_delta::dt_bias":0.5811338739371986,"grad-absmax/layer_09/gated_delta::out_norm":0.0001099901637644507,"param-mean/_layer_33_gated_delta::dt_bias":0.15586811304092407,"grad-absmax/layer_54/moe::shared_expert_fc2":8.692965820955578e-06,"hidden-states-rms/_layer_20_gated_delta::query":0.19259318537512038,"hidden-states-l1/_layer_14_pre_mlp_layernorm":0.0832854012591125,"param-rms/_layer_01_mlp::linear_fc2":0.04623491897163091,"hidden-states-rms/_layer_52_gated_delta::core_attention":0.02317412118089669,"param-l1/_layer_02_gated_delta::out_norm":0.14124298095703125,"hidden-states-mean/_layer_52_input_layernorm":0.009715955000037552,"hidden-states-rms/_layer_11_gated_delta::key":0.14434447549402152,"grad-absmax/layer_51/moe::router":0.00012228563718963414,"grad-absmax/layer_54/gated_delta::conv1d":0.021683186292648315,"hidden-states-rms/_layer_09_gated_delta::key":0.08267946587321294,"hidden-states-absmax/_layer_10_gated_delta::core_attention":0.181640625,"hidden-states-l1/_layer_49_attention::q_layernorm":1.074264058382995,"hidden-states-absmax/_layer_55_gated_delta::g":23.375,"hidden-states-mean/_layer_56_gated_delta::query":-0.014126615420176437,"hidden-states-mean/_layer_45_residual_attn":-0.09786820088245879,"hidden-states-rms/_layer_50_attention::q_layernorm":1.5548315093661444,"update-rms/_layer_29_gated_delta::dt_bias":0.0007856316909538506,"hidden-states-l1/_layer_07_gated_delta::key":0.046116074456700615,"hidden-states-l1/_layer_04_gated_delta::out_norm":0.012306086323390761,"hidden-states-mean/_layer_12_gated_delta::key":0.013582644740658371,"update-mean/_layer_06_gated_delta::out_proj":3.98000139512078e-08,"hidden-states-rms/_layer_20_mlp":0.2163643732594966,"param-rms/_layer_16_gated_delta::out_norm":0.5784358657986943,"hidden-states-l1/_layer_02_gated_delta::query":0.043544487630663826,"hidden-states-rms/_layer_12_gated_delta::value":0.07910671217961271,"param-l1/_layer_28_gated_delta::out_norm":0.795928955078125,"update-absmax/_layer_24_attention::o_proj":0.001953125,"grad-rms/layer_43/moe::router":5.430042075039824e-07,"hidden-states-absmax/_layer_54_gated_delta::g":18.75,"hidden-states-mean/_layer_23_gated_delta::out_proj":-0.0007306470941033476,"hidden-states-mean/_layer_21_mlp":-0.0018481662138973133,"param-absmax/_layer_10_gated_delta::conv1d":0.52734375,"moe/layer_39/act_rms_min":0.08936633914709091,"grad-absmax/layer_55/gated_delta::A_log":7.192783959908411e-05,"hidden-states-rms/_layer_12_gated_delta::key":0.09608395951933187,"param-mean/_layer_03_gated_delta::out_proj":2.4314656911883503e-05,"hidden-states-l1/_layer_26_gated_delta::beta":1.1030985239861637,"update-mean/_layer_34_gated_delta::A_log":-8.84287292137742e-05,"hidden-states-absmax/_layer_30_mlp":79,"hidden-states-absmax/_layer_25_moe::shared_experts":11,"hidden-states-mean/_layer_53_residual_attn":0.12678085288773214,"grad-absmax/layer_30/moe::shared_experts":2.434976886434015e-05,"hidden-states-absmax/_layer_28_moe::routed_experts":20.25,"hidden-states-rms/_layer_50_mlp":1.4636662806016478,"hidden-states-l1/_layer_05_gated_delta::query":0.050937338285128864,"update-l1/_layer_29_gated_delta::A_log":0.0005753673030994833,"hidden-states-absmax/_layer_01_gated_delta::gate":35.5,"hidden-states-mean/_layer_17_mlp":0.0001445089755553276,"hidden-states-l1/_layer_10_moe::router":0.010248827824803642,"hidden-states-l1/_layer_42_mlp":0.31931013695122445,"hidden-states-rms/_layer_19_residual_mlp":1.4283163046924712,"param-l1/_layer_52_gated_delta::conv1d":0.041974037885665894,"hidden-states-rms/_layer_08_gated_delta::in_proj":1.02108948473668,"hidden-states-rms/_layer_16_gated_delta::g":6.346579613839546,"hidden-states-rms/_layer_55_residual_mlp":28.501495414817725,"grad-absmax/layer_11/gated_delta::out_norm":9.331134788226336e-05,"update-absmax/_layer_06_gated_delta::out_proj":0.00390625,"hidden-states-l1/_layer_42_gated_delta::beta":0.9586874793395376,"hidden-states-rms/_layer_54_input_layernorm":0.6635373846754983,"hidden-states-rms/_layer_28_pre_mlp_layernorm":0.17161931117076185,"param-mean/_layer_45_gated_delta::conv1d":-0.000678428856190294,"hidden-states-l1/_layer_08_residual_attn":0.26994621991470097,"grad-rms/layer_26/moe::shared_expert_fc2":1.6801265153469266e-07,"hidden-states-mean/_layer_23_gated_delta::out_norm":7.948440399393784e-05,"hidden-states-rms/_layer_56_input_layernorm":1.0107535111754502,"update-mean/_layer_21_attention::o_proj":1.0274793993403364e-07,"hidden-states-l1/_layer_46_gated_delta::gate":1.42288677965245,"grad-rms/layer_48/moe::router":5.826065077892379e-07,"hidden-states-absmax/_layer_05_gated_delta::beta":11.125,"update-rms/_layer_24_attention::o_proj":0.0001988064655159111,"hidden-states-rms/_layer_17_gated_delta::query":0.2379744838571985,"hidden-states-l1/_layer_01_gated_delta::key":0.10884155140418426,"grad-absmax/layer_42/moe::shared_expert_fc1":1.413126210536575e-05,"grad-absmax/layer_14/gated_delta::out_proj":2.4297811251017265e-05,"hidden-states-mean/_layer_22_pre_mlp_layernorm":-0.0006213355385986612,"grad-absmax/layer_13/moe::expert_fc1":4.813575287698768e-06,"hidden-states-absmax/_layer_41_gated_delta::core_attention":0.1572265625,"update-rms/_layer_45_gated_delta::out_norm":0.0011960399134683486,"hidden-states-absmax/_layer_45_gated_delta::core_attention":0.58203125,"hidden-states-rms/_layer_16_gated_delta::core_attention":0.0012437470581210834,"param-absmax/_layer_08_gated_delta::A_log":1.7550885677337646,"param-mean/_layer_35_gated_delta::A_log":1.0693931579589844,"update-absmax/_layer_27_gated_delta::out_proj":0.001953125,"grad-absmax/layer_24/moe::shared_expert_fc2":1.1427726349211298e-05,"hidden-states-rms/_layer_13_moe::routed_experts":0.04450371892383798,"grad-absmax/layer_08/moe::shared_experts":2.8026968266203767e-06,"update-rms/_layer_36_gated_delta::conv1d":0.000681996647658512,"hidden-states-absmax/_layer_22_mlp":15.125,"hidden-states-rms/_layer_36_gated_delta::gate":1.4482279512726295,"hidden-states-absmax/_layer_27_gated_delta::core_attention":0.080078125,"hidden-states-absmax/_layer_07_pre_mlp_layernorm":2.359375,"update-l1/_layer_05_gated_delta::dt_bias":0.0004058677877765149,"update-absmax/_layer_30_gated_delta::out_proj":0.00390625,"update-l1/_layer_55_gated_delta::conv1d":0.0005071856430731714,"hidden-states-mean/_layer_43_gated_delta::beta":-0.2740434138826625,"param-l1/_layer_06_gated_delta::conv1d":0.03447742760181427,"grad-absmax/layer_39/gated_delta::conv1d":0.0003490518720354885,"hidden-states-l1/_layer_09_pre_mlp_layernorm":0.06393829750279158,"hidden-states-absmax/_layer_42_gated_delta::gate":16.25,"hidden-states-l1/_layer_28_mlp":0.1717741396157908,"hidden-states-l1/_layer_17_gated_delta::key":0.0805416054179198,"param-mean/_layer_36_gated_delta::out_proj":1.1350571185175795e-06,"hidden-states-absmax/_layer_49_mlp":75.5,"hidden-states-absmax/_layer_08_mlp":11,"grad-absmax/layer_06/moe::router":6.32144947303459e-05,"hidden-states-l1/_layer_03_input_layernorm":0.17752116054548747,"hidden-states-mean/_layer_35_moe::routed_experts":-0.00026449410360990385,"update-absmax/_layer_43_gated_delta::A_log":0.0013040900230407715,"grad-absmax/layer_08/moe::expert_fc2":8.672033686707437e-07,"param-rms/_layer_04_gated_delta::out_proj":0.04559399962945179,"hidden-states-rms/_layer_36_gated_delta::g":5.365527797754858,"hidden-states-l1/_layer_31_gated_delta::out_norm":0.064735025433901,"param-mean/_layer_37_gated_delta::out_norm":1.19329833984375,"grad-absmax/layer_44/gated_delta::A_log":5.917954331380315e-05,"update-rms/_layer_55_gated_delta::conv1d":0.0006696314847765103,"grad-rms/layer_06/gated_delta::out_proj":4.744006657939076e-07,"update-rms/gated_delta::out_proj":0.0002058149976397235,"hidden-states-absmax/_layer_53_pre_mlp_layernorm":13.1875,"grad-rms/layer_05/gated_delta::conv1d":8.421537066480153e-06,"hidden-states-l1/_layer_51_gated_delta::g":8.198792481278478,"hidden-states-absmax/_layer_09_gated_delta::g":24.25,"update-l1/_layer_46_gated_delta::out_norm":0.00091552734375,"update-rms/_layer_32_gated_delta::dt_bias":0.0007705659338098143,"grad-rms/layer_08/moe::shared_expert_fc2":1.3258676285223995e-07,"hidden-states-rms/_layer_49_input_layernorm":0.770265110663488,"hidden-states-mean/_layer_49_attention::q_layernorm":0.09592387478636202,"lm loss":0.60963374376297,"update-absmax/_layer_52_gated_delta::dt_bias":0.0016918778419494629,"hidden-states-rms/_layer_22_moe::routed_experts":0.09212521797918194,"param-mean/_layer_11_gated_delta::A_log":1.0395716428756714,"hidden-states-rms/_layer_22_attention::q_layernorm":1.6696041331389138,"hidden-states-l1/_layer_47_residual_mlp":3.7562195964121337,"hidden-states-l1/_layer_43_gated_delta::beta":1.070202478194532,"hidden-states-l1/_layer_09_residual_mlp":0.30344373624846177,"hidden-states-l1/_layer_19_residual_attn":0.604451132351295,"grad-rms/layer_25/moe::shared_expert_fc2":2.0178256557852796e-07,"hidden-states-rms/_layer_01_gated_delta::gate":2.556078016971673,"moe/layer_32/act_rms_min":0.05384457856416702,"grad-absmax/layer_17/gated_delta::out_proj":5.0460195780033246e-05,"grad-absmax/layer_06/moe::expert_fc2":9.182117537420709e-06,"hidden-states-mean/_layer_23_gated_delta::gate":0.23205780769083592,"hidden-states-mean/_layer_32_moe::router":0.010227828360053438,"hidden-states-absmax/_layer_09_moe::router":0.9966027140617371,"update-absmax/_layer_30_gated_delta::conv1d":0.00390625,"hidden-states-mean/_layer_50_attention::o_proj":0.013899003881097303,"hidden-states-mean/_layer_47_mlp":-0.002216993629723028,"update-rms/_layer_48_gated_delta::conv1d":0.0006949912622485542,"param-rms/_layer_33_gated_delta::A_log":1.0826831052237653,"hidden-states-mean/_layer_32_pre_mlp_layernorm":3.1645635452020294e-05,"hidden-states-l1/_layer_21_attention::k_layernorm":2.687734209028984,"hidden-states-absmax/_layer_34_gated_delta::out_proj":31.125,"hidden-states-absmax/_layer_33_gated_delta::core_attention":0.65625,"hidden-states-mean/_layer_12_moe::router":0.010257124047616965,"hidden-states-l1/_layer_22_input_layernorm":0.33731077162155765,"moe/layer_09/act_rms_max":0.06034287437796593,"hidden-states-absmax/_layer_37_gated_delta::key":3.34375,"hidden-states-rms/_layer_38_gated_delta::beta":1.4059375829864298,"hidden-states-absmax/_layer_33_residual_mlp":502,"grad-absmax/layer_33/gated_delta::out_norm":2.7882124413736165e-05,"update-l1/_layer_33_gated_delta::dt_bias":0.0006376823293976486,"param-absmax/_layer_41_gated_delta::conv1d":1.0078125,"hidden-states-absmax/_layer_50_mlp":198,"grad-rms/layer_39/gated_delta::dt_bias":2.0006553295108543e-06,"update-rms/_layer_01_gated_delta::out_proj":0.00018541357214884409,"grad-absmax/layer_29/gated_delta::A_log":5.7174215726263355e-06,"hidden-states-mean/_layer_16_moe::routed_experts":-3.5706297827308467e-06,"hidden-states-rms/_layer_44_gated_delta::in_proj":1.8484830683011244,"hidden-states-rms/_layer_24_residual_attn":2.210546583967126,"hidden-states-absmax/_layer_27_gated_delta::out_norm":20.5,"hidden-states-rms/_layer_37_mlp":0.3482036988638254,"hidden-states-rms/_layer_12_gated_delta::out_proj":0.18672260531521964,"param-l1/embedding":0.28660061955451965,"update-rms/_layer_11_gated_delta::out_norm":0.001072900136184506,"grad-rms/layer_23/moe::expert_fc1":1.8549519844188352e-08,"hidden-states-rms/_layer_45_gated_delta::key":0.26450307477727697,"hidden-states-absmax/_layer_28_mlp":20.125,"hidden-states-l1/_layer_07_gated_delta::out_norm":0.016717022415044594,"param-absmax/_layer_42_gated_delta::out_norm":1.8203125,"hidden-states-l1/_layer_43_gated_delta::in_proj":1.1764005662329937,"hidden-states-mean/_layer_01_gated_delta::in_proj":-0.4055615088909299,"hidden-states-rms/_layer_04_gated_delta::core_attention":0.0017935257013078276,"update-mean/_layer_26_gated_delta::out_proj":-1.827950768529263e-08,"hidden-states-absmax/_layer_21_attention::q_layernorm":17.5,"update-absmax/_layer_22_attention::linear_qkv":0.001953125,"hidden-states-l1/gated_delta::value":0.048031710796341186,"hidden-states-absmax/_layer_13_moe::routed_experts":34.5,"grad-absmax/layer_53/gated_delta::out_norm":6.690932787023485e-05,"hidden-states-mean/_layer_02_gated_delta::in_proj":-0.03509813804137977,"hidden-states-absmax/_layer_12_gated_delta::beta":15.5625,"update-rms/_layer_51_gated_delta::conv1d":0.0007931014729221531,"hidden-states-mean/_layer_45_gated_delta::gate":-0.555784172509214,"param-absmax/_layer_23_gated_delta::out_norm":1.40625,"grad-absmax/layer_06/moe::shared_experts":4.401959358801832e-06,"hidden-states-l1/_layer_15_gated_delta::beta":2.0123655810791945,"hidden-states-l1/_layer_01_residual_attn":0.21948147946375363,"moe/global/act_min_to_median_min":0,"hidden-states-rms/_layer_25_gated_delta::gate":1.2443905637002441,"hidden-states-mean/_layer_33_gated_delta::in_proj":-0.29418390007338346,"hidden-states-l1/_layer_27_gated_delta::query":0.06857810241915016,"hidden-states-rms/_layer_32_gated_delta::key":0.2971857529195485,"param-absmax/_layer_02_mlp::linear_fc2":0.65625,"update-absmax/_layer_15_gated_delta::dt_bias":0.0025386810302734375,"param-rms/_layer_32_gated_delta::dt_bias":0.25572241692009945,"hidden-states-l1/_layer_33_gated_delta::gate":1.127156034368996,"grad-rms/layer_08/moe::shared_experts":6.805687108375035e-07,"hidden-states-l1/_layer_23_gated_delta::query":0.08470645066576608,"hidden-states-mean/_layer_48_gated_delta::out_proj":-6.345850880973892e-05,"hidden-states-mean/_layer_45_gated_delta::core_attention":0.0007485362312874176,"hidden-states-rms/_layer_07_gated_delta::beta":2.114356043567456,"hidden-states-l1/_layer_09_gated_delta::in_proj":0.807082075621247,"param-l1/_layer_34_gated_delta::conv1d":0.0373600572347641,"hidden-states-rms/_layer_07_gated_delta::out_norm":0.036420387154840396,"param-absmax/_layer_27_gated_delta::A_log":1.6375707387924194,"update-rms/_layer_06_gated_delta::A_log":0.0005063573949560951,"grad-absmax/layer_56/moe::shared_expert_fc1":8.879535016603768e-06,"hidden-states-mean/_layer_45_gated_delta::query":0.08665672232588796,"param-mean/_layer_53_gated_delta::dt_bias":0.6845307946205139,"hidden-states-rms/_layer_53_moe::routed_experts":0.35913184265547055,"param-mean/_layer_27_gated_delta::dt_bias":-0.12972182035446167,"hidden-states-absmax/_layer_31_gated_delta::value":8.5,"update-mean/_layer_36_gated_delta::out_proj":2.9303665272095714e-08,"grad-absmax/layer_13/moe::shared_experts":5.3286416914488655e-06,"hidden-states-mean/_layer_14_gated_delta::g":-7.771724673715195,"hidden-states-l1/_layer_25_gated_delta::core_attention":0.00173201944837206,"update-absmax/_layer_19_gated_delta::conv1d":0.0029296875,"grad-absmax/layer_44/moe::expert_fc2":4.136805728194304e-06,"hidden-states-absmax/_layer_06_gated_delta::out_norm":30.5,"hidden-states-l1/_layer_12_gated_delta::key":0.05163311083711515,"update-rms/_layer_22_attention::linear_qkv":0.00018255897580549704,"grad-absmax/layer_12/moe::shared_expert_fc1":2.9670040021301247e-05,"hidden-states-l1/_layer_35_gated_delta::query":0.13162027378323957,"hidden-states-l1/_layer_54_gated_delta::query":0.08063070692190015,"param-mean/_layer_27_gated_delta::A_log":0.737655520439148,"update-l1/_layer_27_gated_delta::out_proj":0.00014756446762476116,"hidden-states-absmax/_layer_51_moe::router":0.9998231530189514,"hidden-states-l1/_layer_15_gated_delta::out_proj":0.0938828550437516,"update-l1/_layer_50_attention::o_proj":0.0001925734686665237,"hidden-states-l1/_layer_12_moe::router":0.010257124047616965,"hidden-states-l1/_layer_36_moe::routed_experts":0.07430921857920386,"hidden-states-mean/_layer_47_attention::k_layernorm":-0.19228104558957165,"hidden-states-l1/_layer_07_moe::router":0.01024086191111595,"hidden-states-l1/_layer_18_gated_delta::core_attention":0.0003723638808937527,"hidden-states-rms/_layer_37_input_layernorm":0.4543630138802012,"hidden-states-l1/_layer_49_input_layernorm":0.31145689624034123,"grad-rms/layer_41/moe::expert_fc1":1.977370669050663e-08,"param-absmax/_layer_15_gated_delta::A_log":2.173556327819824,"hidden-states-absmax/_layer_39_gated_delta::in_proj":26.375,"hidden-states-rms/_layer_25_gated_delta::in_proj":1.4473059617096395,"hidden-states-rms/_layer_36_gated_delta::core_attention":0.0038976871142789114,"grad-rms/layer_15/moe::expert_fc2":2.379433731103687e-08,"grad-rms/layer_56/gated_delta::dt_bias":6.672153523098729e-06,"moe/layer_33/act_rms_max":0.2985682785511017,"hidden-states-absmax/_layer_14_gated_delta::out_proj":16.625,"hidden-states-absmax/_layer_48_mlp":94,"hidden-states-rms/_layer_26_gated_delta::out_norm":0.16028728384091476,"param-l1/_layer_17_gated_delta::A_log":0.9889010190963745,"param-mean/_layer_21_attention::o_proj":-2.5437588192289695e-05,"hidden-states-mean/_layer_41_gated_delta::out_proj":-0.0015435262560577697,"param-rms/_layer_17_gated_delta::out_proj":0.043123777014125196,"update-mean/_layer_06_gated_delta::A_log":2.2102962248027325e-05,"update-l1/_layer_51_gated_delta::out_proj":0.00014238337462302297,"moe/global/act_min_to_median_lt_0_2_ratio":0.037037037037037035,"hidden-states-absmax/_layer_54_gated_delta::out_proj":165,"hidden-states-mean/_layer_32_residual_mlp":-0.04022499280869875,"param-mean/_layer_06_gated_delta::out_proj":-2.1372034098021686e-05,"hidden-states-l1/gated_delta::gate":1.0594373112124527,"hidden-states-absmax/_layer_44_gated_delta::g":17.25,"hidden-states-rms/_layer_03_pre_mlp_layernorm":0.04953360942711809,"update-l1/_layer_20_gated_delta::out_proj":0.00013673781359102577,"hidden-states-l1/_layer_06_moe::shared_experts":0.030780252013236524,"hidden-states-absmax/_layer_30_gated_delta::key":6.78125,"hidden-states-absmax/_layer_38_moe::shared_experts":39,"hidden-states-l1/_layer_51_residual_attn":4.565705843651991,"update-rms/_layer_40_gated_delta::out_norm":0.0016558422810364424,"hidden-states-absmax/_layer_06_gated_delta::query":9.9375,"hidden-states-rms/_layer_44_gated_delta::value":0.13285205146664464,"grad-rms/layer_09/moe::shared_expert_fc1":2.631682524110494e-07,"moe/layer_25/act_rms_median":0.08520582318305969,"hidden-states-mean/_layer_42_gated_delta::value":-0.0002850087233643202,"hidden-states-mean/_layer_25_pre_mlp_layernorm":-0.0006420581668461164,"update-mean/_layer_45_gated_delta::conv1d":-2.033403143286705e-05,"hidden-states-l1/_layer_27_gated_delta::beta":1.709867442830496,"grad-absmax/layer_28/moe::expert_fc1":4.2192486944259144e-06,"param-absmax/gated_delta::out_norm":2.609375,"hidden-states-mean/_layer_16_gated_delta::gate":-0.17320390620278195,"hidden-states-rms/_layer_54_gated_delta::gate":1.8657043018723931,"hidden-states-mean/_layer_50_attention::k_layernorm":-0.01007847514939012,"update-absmax/_layer_55_gated_delta::out_proj":0.00390625,"grad-rms/layer_33/moe::shared_experts":1.8857246913854041e-06,"param-absmax/_layer_01_gated_delta::dt_bias":1.4029160737991333,"hidden-states-absmax/_layer_02_gated_delta::g":16.75,"hidden-states-rms/_layer_26_gated_delta::out_proj":0.3682346242938278,"hidden-states-mean/_layer_07_moe::shared_experts":2.274222695711539e-05,"hidden-states-l1/_layer_14_gated_delta::gate":0.8225418117692246,"grad-absmax/layer_05/moe::expert_fc2":1.2675549442064948e-05,"grad-rms/gated_delta::A_log":2.1647797808188482e-05,"hidden-states-rms/_layer_47_input_layernorm":0.8194026203447311,"param-mean/_layer_30_gated_delta::conv1d":0.0008834038162603974,"param-l1/_layer_13_attention::linear_qkv":0.02960268035531044,"hidden-states-rms/_layer_23_gated_delta::core_attention":0.002394432413131806,"update-mean/_layer_34_gated_delta::dt_bias":-7.396106229862198e-05,"hidden-states-rms/_layer_04_residual_mlp":0.5803416092921704,"hidden-states-mean/_layer_44_moe::router":0.010180323450616749,"grad-rms/layer_54/moe::shared_expert_fc2":9.283296039421777e-08,"hidden-states-mean/_layer_08_residual_mlp":0.0011626231938747398,"hidden-states-absmax/_layer_12_input_layernorm":25.875,"param-mean/_layer_50_attention::linear_qkv":0.0001288944622501731,"hidden-states-mean/_layer_34_moe::router":0.010208949827583604,"update-rms/_layer_34_gated_delta::conv1d":0.0007855012986300101,"hidden-states-l1/_layer_04_gated_delta::core_attention":0.0006563290571888157,"hidden-states-rms/_layer_37_gated_delta::value":0.08772295208826265,"update-l1/_layer_10_gated_delta::out_norm":0.00048828125,"hidden-states-mean/_layer_44_residual_attn":-0.0933643839793663,"hidden-states-rms/_layer_34_gated_delta::core_attention":0.004986473211050211,"hidden-states-rms/_layer_33_residual_attn":5.944857920501475,"update-absmax/_layer_42_gated_delta::dt_bias":0.001639723777770996,"update-mean/_layer_40_gated_delta::out_proj":4.03649096369918e-08,"hidden-states-l1/_layer_04_gated_delta::in_proj":0.6913183886980965,"hidden-states-mean/_layer_14_gated_delta::query":0.022794400265145332,"hidden-states-absmax/_layer_53_gated_delta::in_proj":31.125,"hidden-states-absmax/_layer_06_gated_delta::value":5.5625,"hidden-states-l1/_layer_25_moe::shared_experts":0.1121863782031987,"hidden-states-rms/_layer_03_moe::routed_experts":0.018009869391017144,"hidden-states-mean/_layer_55_gated_delta::core_attention":9.093370674132785e-05,"param-rms/_layer_48_gated_delta::A_log":1.4576580754924036,"hidden-states-absmax/_layer_09_input_layernorm":11.875,"hidden-states-l1/_layer_45_gated_delta::g":6.686087928544036,"hidden-states-mean/_layer_55_mlp":-0.00016025679815827884,"hidden-states-absmax/_layer_41_moe::router":0.9998669624328613,"hidden-states-absmax/_layer_45_moe::shared_experts":69.5,"hidden-states-l1/_layer_14_gated_delta::value":0.030708411643911797,"update-mean/_layer_20_gated_delta::A_log":5.5889482609927654e-05,"hidden-states-mean/_layer_17_moe::routed_experts":-3.0333722029371096e-06,"hidden-states-mean/_layer_35_gated_delta::out_proj":-0.004182925480803905,"hidden-states-absmax/_layer_37_pre_mlp_layernorm":3.96875,"grad-rms/layer_13/moe::shared_experts":9.301684747311228e-07,"grad-absmax/layer_19/moe::expert_fc2":2.6757552404887974e-06,"param-absmax/_layer_31_gated_delta::dt_bias":0.7081289291381836,"hidden-states-mean/_layer_56_gated_delta::core_attention":-0.0003726072471530434,"grad-rms/layer_09/gated_delta::dt_bias":1.8662593072234212e-06,"hidden-states-absmax/_layer_02_gated_delta::key":1.3984375,"hidden-states-rms/_layer_50_moe::router":0.07216878508256834,"hidden-states-rms/_layer_49_residual_mlp":10.263178711475502,"hidden-states-mean/_layer_27_residual_mlp":-0.020649476575789045,"grad-rms/layer_25/moe::expert_fc1":1.9342802830816005e-08,"grad-rms/layer_34/moe::shared_expert_fc1":1.96544238359366e-07,"update-l1/_layer_27_gated_delta::conv1d":0.000646504748146981,"hidden-states-l1/_layer_37_moe::shared_experts":0.09873284086380492,"param-absmax/_layer_46_gated_delta::out_norm":1.8515625,"grad-rms/layer_45/moe::shared_expert_fc2":2.9838809443166047e-07,"param-mean/_layer_16_gated_delta::out_norm":0.5567626953125,"hidden-states-absmax/_layer_22_moe::shared_experts":8.625,"update-l1/_layer_23_gated_delta::A_log":0.0005283989012241364,"hidden-states-mean/_layer_52_gated_delta::g":-7.554796642636853,"update-l1/gated_delta::dt_bias":0.0005097991967459723,"grad-rms/layer_55/gated_delta::A_log":1.398511587686668e-05,"hidden-states-rms/_layer_35_moe::routed_experts":0.1579101200091427,"hidden-states-rms/_layer_07_gated_delta::out_proj":0.08217604587456474,"hidden-states-rms/_layer_08_residual_mlp":0.7238483410861991,"hidden-states-mean/_layer_56_moe::routed_experts":-4.839529219152657e-07,"update-l1/_layer_12_gated_delta::dt_bias":0.0004152142792008817,"param-rms/_layer_12_gated_delta::conv1d":0.05887010689554295,"update-absmax/_layer_29_gated_delta::out_proj":0.001953125,"hidden-states-mean/_layer_39_input_layernorm":0.003502865506677688,"update-mean/_layer_35_gated_delta::dt_bias":-9.565416985424235e-05,"param-l1/_layer_14_gated_delta::out_norm":0.3956756591796875,"update-rms/_layer_27_gated_delta::conv1d":0.0008271128471360616,"hidden-states-l1/_layer_28_residual_attn":1.3550036376719043,"update-rms/_layer_52_gated_delta::out_proj":0.0001998042112079126,"param-rms/_layer_42_gated_delta::dt_bias":0.3085748690852982,"hidden-states-mean/_layer_37_residual_mlp":-0.0803356042904382,"param-l1/_layer_47_attention::linear_qkv":0.027755603194236755,"hidden-states-l1/_layer_45_moe::router":0.010212490095693286,"hidden-states-rms/_layer_06_gated_delta::g":6.73137763876796,"hidden-states-mean/_layer_12_gated_delta::in_proj":-0.08316988609878033,"param-rms/_layer_16_gated_delta::out_proj":0.043826931875141736,"hidden-states-rms/_layer_12_gated_delta::gate":1.3379047383025544,"grad-rms/layer_06/gated_delta::conv1d":0.0002531397965789762,"update-absmax/_layer_08_gated_delta::conv1d":0.0030364990234375,"param-l1/_layer_42_gated_delta::out_norm":1.200439453125,"grad-rms/layer_46/moe::expert_fc1":1.841373833304567e-08,"hidden-states-absmax/_layer_13_pre_mlp_layernorm":6.375,"hidden-states-mean/_layer_28_gated_delta::in_proj":-0.08072196873059391,"hidden-states-rms/_layer_42_residual_mlp":8.002132608611685,"update-absmax/_layer_31_gated_delta::out_norm":0.0078125,"hidden-states-absmax/_layer_31_gated_delta::query":23.25,"hidden-states-absmax/_layer_20_moe::routed_experts":6.09375,"grad-absmax/layer_31/moe::router":5.768088885815814e-05,"grad-absmax/layer_49/moe::shared_expert_fc1":2.1640158593072556e-05,"grad-absmax/layer_38/gated_delta::out_norm":2.6772824639920145e-05,"param-absmax/_layer_04_gated_delta::out_proj":0.5,"grad-rms/layer_20/moe::shared_experts":2.126370567272612e-06,"hidden-states-absmax/_layer_43_gated_delta::in_proj":38.5,"param-absmax/_layer_01_gated_delta::out_proj":0.7890625,"hidden-states-mean/_layer_40_moe::router":0.010084047653722391,"hidden-states-absmax/_layer_19_gated_delta::gate":9.5625,"hidden-states-rms/_layer_50_moe::routed_experts":0.34674951351936273,"param-l1/_layer_53_gated_delta::conv1d":0.03444461524486542,"hidden-states-mean/_layer_28_gated_delta::key":0.006732066033132795,"moe/layer_21/act_rms_min_to_median":0.42950192156733524,"hidden-states-absmax/_layer_48_gated_delta::gate":25.875,"hidden-states-mean/_layer_38_gated_delta::out_norm":-0.0012874782921720495,"grad-rms/layer_03/moe::expert_fc1":7.640038733405704e-09,"update-rms/_layer_14_gated_delta::dt_bias":0.0006097721911412806,"hidden-states-l1/_layer_46_gated_delta::value":0.06474798199510359,"update-l1/_layer_44_gated_delta::conv1d":0.0005800735088996589,"hidden-states-l1/_layer_56_moe::shared_experts":0.4314485110579146,"hidden-states-mean/_layer_10_moe::shared_experts":-0.0001623400796945488,"hidden-states-l1/_layer_54_moe::shared_experts":0.14869602307033813,"hidden-states-l1/_layer_39_residual_attn":2.418990900396216,"param-l1/_layer_43_gated_delta::conv1d":0.040200185030698776,"grad-rms/layer_10/moe::shared_experts":8.34769099986306e-07,"update-mean/_layer_16_gated_delta::conv1d":8.003534276213031e-06,"hidden-states-l1/_layer_32_gated_delta::query":0.27147332089821347,"hidden-states-absmax/residual_attn":2816,"hidden-states-l1/_layer_28_gated_delta::in_proj":0.804874448882157,"hidden-states-rms/_layer_12_moe::routed_experts":0.03520195372559519,"param-mean/_layer_08_gated_delta::out_norm":0.3342742919921875,"hidden-states-l1/_layer_32_gated_delta::key":0.19965512450398498,"grad-absmax/layer_13/moe::router":5.9626621805364266e-05,"hidden-states-absmax/_layer_21_input_layernorm":34.75,"param-l1/_layer_41_gated_delta::conv1d":0.037029776722192764,"hidden-states-l1/_layer_19_residual_mlp":0.6341449778252813,"hidden-states-absmax/_layer_29_gated_delta::out_norm":33.5,"hidden-states-l1/_layer_55_gated_delta::beta":3.149373906776179,"hidden-states-absmax/_layer_04_gated_delta::out_norm":15.5625,"param-l1/_layer_42_gated_delta::dt_bias":0.2522984743118286,"hidden-states-absmax/_layer_19_residual_attn":494,"grad-absmax/layer_43/gated_delta::out_proj":0.00010703969746828079,"update-rms/_layer_02_mlp::linear_fc1":0.00022418917799906673,"moe/layer_38/act_rms_min_to_median":0.560351987135282,"update-l1/_layer_34_gated_delta::out_norm":0.000640869140625,"hidden-states-absmax/_layer_13_input_layernorm":17,"hidden-states-mean/_layer_03_gated_delta::gate":-0.09336521099391115,"param-rms/_layer_27_gated_delta::dt_bias":0.24910934312862462,"grad-rms/layer_15/moe::shared_experts":8.951619139273173e-07,"update-l1/_layer_36_gated_delta::out_proj":0.0001316530251642689,"hidden-states-rms/_layer_28_residual_mlp":3.815690605972581,"hidden-states-mean/_layer_39_gated_delta::out_norm":0.0002921427133439736,"param-rms/_layer_07_gated_delta::dt_bias":0.31742007697025937,"hidden-states-absmax/_layer_40_residual_attn":502,"hidden-states-absmax/_layer_35_input_layernorm":30.75,"hidden-states-rms/_layer_54_gated_delta::query":0.12133233686803238,"param-absmax/_layer_03_gated_delta::out_proj":0.67578125,"hidden-states-mean/_layer_48_gated_delta::core_attention":0.00042888180925219465,"hidden-states-l1/_layer_48_mlp":0.5283921394815612,"update-mean/_layer_27_gated_delta::conv1d":-1.33069643197814e-05,"param-rms/_layer_41_gated_delta::dt_bias":0.3683206038231648,"hidden-states-mean/_layer_40_gated_delta::query":0.050719863262160214,"hidden-states-mean/_layer_22_attention::k_layernorm":0.07622311427868775,"hidden-states-rms/_layer_39_gated_delta::g":4.4237035679967835,"hidden-states-l1/_layer_23_gated_delta::out_proj":0.2847857516748609,"hidden-states-absmax/_layer_39_moe::shared_experts":18.25,"hidden-states-mean/_layer_17_gated_delta::out_proj":-0.0004975421481629786,"hidden-states-rms/_layer_14_gated_delta::in_proj":1.361067093550599,"hidden-states-l1/_layer_17_moe::routed_experts":0.033367671956128164,"param-rms/_layer_23_gated_delta::dt_bias":0.2055725586031674,"moe/layer_11/act_rms_median":0.03293673321604729,"hidden-states-rms/_layer_12_gated_delta::out_norm":0.08046262489023957,"hidden-states-l1/_layer_05_gated_delta::g":3.309071458467419,"hidden-states-l1/_layer_28_gated_delta::gate":0.8252874404534627,"grad-absmax/layer_14/moe::shared_expert_fc2":3.3855653782666195e-06,"param-l1/_layer_12_gated_delta::dt_bias":0.3784380257129669,"hidden-states-l1/_layer_23_gated_delta::g":5.925978413340856,"hidden-states-mean/_layer_10_gated_delta::value":0.0007285236783800933,"grad-absmax/layer_42/gated_delta::out_proj":0.00011846545385196805,"hidden-states-l1/_layer_05_gated_delta::out_proj":0.053524530354086206,"grad-rms/layer_33/moe::shared_expert_fc1":2.0682830773535992e-07,"hidden-states-absmax/_layer_34_input_layernorm":34.5,"moe/layer_53/act_rms_min_to_median":0.5717034284286034,"param-mean/_layer_41_gated_delta::A_log":0.919868528842926,"hidden-states-mean/_layer_39_residual_mlp":-0.08514355392975258,"update-absmax/_layer_22_attention::o_proj":0.001953125,"update-l1/gated_delta::conv1d":0.0006166717129027737,"grad-rms/layer_30/gated_delta::conv1d":1.1798915957469448e-05,"hidden-states-absmax/moe::shared_experts":560,"hidden-states-rms/_layer_03_input_layernorm":0.3321250756584648,"update-rms/_layer_02_gated_delta::conv1d":0.0008210011190888416,"update-absmax/_layer_44_gated_delta::out_proj":0.001953125,"hidden-states-mean/_layer_03_gated_delta::out_norm":4.371829502837897e-05,"hidden-states-absmax/_layer_52_gated_delta::query":32.75,"update-rms/_layer_44_gated_delta::conv1d":0.0007475010682950511,"hidden-states-l1/_layer_14_residual_mlp":0.4696805440998726,"update-rms/_layer_25_gated_delta::out_proj":0.00020952909719235366,"hidden-states-rms/_layer_51_gated_delta::key":0.28163294773872993,"hidden-states-rms/_layer_35_residual_attn":6.259793725722989,"hidden-states-rms/_layer_51_moe::shared_experts":1.0981349352494276,"hidden-states-absmax/_layer_12_gated_delta::query":6.6875,"hidden-states-absmax/_layer_15_mlp":15.0625,"param-l1/_layer_48_gated_delta::out_norm":1.5618896484375,"update-l1/_layer_48_gated_delta::out_proj":0.00013444326759781688,"hidden-states-rms/_layer_17_residual_mlp":1.337505414939362,"hidden-states-absmax/_layer_55_gated_delta::key":5.5625,"param-rms/_layer_15_gated_delta::out_proj":0.044530971425841503,"update-absmax/_layer_37_gated_delta::out_proj":0.001953125,"grad-rms/layer_12/moe::expert_fc2":1.962423711576201e-08,"hidden-states-absmax/_layer_13_mlp":34,"hidden-states-l1/_layer_55_gated_delta::key":0.07620582892964231,"grad-absmax/layer_27/moe::shared_expert_fc2":2.522588374631596e-06,"hidden-states-rms/_layer_21_residual_mlp":1.6882260085412861,"grad-absmax/layer_05/gated_delta::conv1d":0.00016466208035126328,"hidden-states-rms/_layer_46_gated_delta::in_proj":2.1279417582480376,"grad-absmax/layer_25/moe::expert_fc2":1.130452801589854e-06,"hidden-states-absmax/_layer_12_gated_delta::out_proj":21,"hidden-states-l1/_layer_19_gated_delta::gate":0.8049581414411237,"hidden-states-rms/_layer_26_moe::routed_experts":0.10372409542171662,"grad-rms/layer_18/gated_delta::dt_bias":4.558567049405908e-06,"hidden-states-absmax/_layer_11_moe::router":0.9991276860237122,"hidden-states-rms/_layer_45_residual_attn":8.45335474924885,"param-absmax/_layer_25_gated_delta::dt_bias":0.47057482600212097,"hidden-states-rms/_layer_40_moe::router":0.07216878508256834,"grad-absmax/layer_28/gated_delta::dt_bias":1.569710366311483e-05,"hidden-states-l1/_layer_11_gated_delta::gate":0.8417506029308561,"hidden-states-rms/_layer_51_gated_delta::g":8.86124621429613,"hidden-states-l1/_layer_42_gated_delta::key":0.05529129419223001,"hidden-states-mean/_layer_34_gated_delta::core_attention":0.0002775313663520058,"hidden-states-rms/_layer_45_gated_delta::gate":2.1637133562955078,"grad-rms/layer_16/moe::shared_expert_fc2":2.067386412314881e-07,"hidden-states-mean/_layer_10_gated_delta::gate":-0.25796040987496494,"grad-rms/layer_04/moe::expert_fc1":7.937744439933234e-09,"param-rms/mlp::linear_fc2":0.0461965439602632,"hidden-states-mean/_layer_48_moe::router":0.010241028307442395,"hidden-states-absmax/_layer_17_gated_delta::key":6.46875,"moe/layer_04/act_rms_max":0.05664895474910736,"hidden-states-l1/_layer_05_moe::router":0.010305154223069227,"param-absmax/_layer_05_gated_delta::dt_bias":0.6797031164169312,"update-absmax/_layer_19_gated_delta::out_norm":0.0078125,"hidden-states-absmax/_layer_05_gated_delta::out_proj":5.46875,"grad-absmax/layer_16/moe::expert_fc1":1.6272853144982946e-06,"hidden-states-mean/_layer_13_mlp":0.0006342666630504028,"hidden-states-rms/_layer_20_gated_delta::value":0.06752567821940915,"hidden-states-mean/_layer_37_gated_delta::out_proj":-0.0019563245590865086,"hidden-states-l1/_layer_43_moe::routed_experts":0.12087266388838283,"hidden-states-mean/_layer_30_gated_delta::out_norm":-0.002359141876893267,"param-rms/_layer_37_gated_delta::dt_bias":0.4429876979428628,"param-l1/_layer_55_gated_delta::out_norm":1.368896484375,"param-absmax/_layer_48_gated_delta::out_norm":2.09375,"param-l1/_layer_56_gated_delta::conv1d":0.0353362113237381,"grad-absmax/layer_38/moe::expert_fc1":9.236662663170137e-06,"update-absmax/_layer_03_gated_delta::out_proj":0.001953125,"hidden-states-mean/_layer_15_gated_delta::core_attention":1.5110650547815739e-05,"param-l1/_layer_32_gated_delta::out_norm":0.327484130859375,"hidden-states-absmax/_layer_03_moe::routed_experts":3.546875,"param-mean/_layer_26_gated_delta::A_log":0.9509900808334351,"update-mean/_layer_30_gated_delta::conv1d":-3.1404237233800814e-05,"grad-absmax/layer_33/moe::shared_expert_fc1":2.6602572688716464e-05,"grad-absmax/layer_46/moe::shared_expert_fc1":8.953869109973311e-05,"hidden-states-rms/_layer_40_gated_delta::gate":1.8860462816993548,"hidden-states-rms/_layer_01_gated_delta::in_proj":3.82149170052481,"hidden-states-absmax/_layer_33_pre_mlp_layernorm":4.03125,"hidden-states-rms/_layer_52_moe::routed_experts":0.24979564310971342,"param-mean/_layer_28_gated_delta::dt_bias":0.1259310394525528,"update-mean/_layer_23_gated_delta::out_proj":1.408555903026354e-07,"update-l1/_layer_39_gated_delta::out_norm":0.0003662109375,"grad-rms/layer_19/gated_delta::conv1d":2.496890136825667e-05,"moe/layer_11/act_rms_max":0.06236685439944267,"grad-absmax/layer_45/gated_delta::out_norm":4.8561480070929974e-05,"hidden-states-absmax/_layer_12_gated_delta::value":2.90625,"hidden-states-absmax/gated_delta::core_attention":5.75,"param-rms/_layer_55_gated_delta::A_log":1.8207000188958566,"update-rms/_layer_17_gated_delta::dt_bias":0.0006088437281097042,"param-l1/_layer_10_gated_delta::A_log":1.2288111448287964,"hidden-states-mean/_layer_45_gated_delta::in_proj":-0.2511976737691313,"hidden-states-absmax/_layer_52_gated_delta::value":12.3125,"hidden-states-rms/_layer_02_gated_delta::beta":1.8422342760014199,"update-rms/_layer_34_gated_delta::out_proj":0.00020434995011988006,"hidden-states-rms/_layer_23_gated_delta::in_proj":1.261914009782331,"hidden-states-mean/_layer_15_gated_delta::key":0.01893181952021355,"grad-absmax/layer_38/gated_delta::dt_bias":2.8877628210466355e-05,"hidden-states-absmax/_layer_07_gated_delta::g":19.625,"hidden-states-l1/_layer_02_gated_delta::in_proj":0.7868387468916102,"hidden-states-absmax/_layer_39_gated_delta::gate":19.875,"hidden-states-l1/_layer_51_gated_delta::out_proj":1.9431986741064398,"grad-rms/layer_28/moe::shared_experts":1.568895460880448e-06,"hidden-states-mean/_layer_29_residual_mlp":-0.02393293293511068,"hidden-states-rms/_layer_22_attention::o_proj":0.3924182401518876,"hidden-states-mean/_layer_30_gated_delta::value":-0.0001900086434987433,"hidden-states-l1/_layer_55_gated_delta::query":0.10502555728263346,"param-rms/_layer_40_gated_delta::A_log":1.3291822376754225,"hidden-states-absmax/_layer_19_gated_delta::key":4.78125,"hidden-states-mean/_layer_02_gated_delta::g":-3.815148331540624,"hidden-states-rms/_layer_51_gated_delta::core_attention":0.01314670449120609,"param-l1/_layer_54_gated_delta::dt_bias":0.6362190842628479,"param-l1/_layer_39_gated_delta::A_log":1.1825588941574097,"samples vs steps":76288,"update-absmax/_layer_40_gated_delta::conv1d":0.0035247802734375,"hidden-states-l1/_layer_40_gated_delta::out_norm":0.08506880969629403,"grad-rms/layer_05/gated_delta::dt_bias":1.4266516012436254e-06,"hidden-states-rms/_layer_12_gated_delta::query":0.12967704360408414,"hidden-states-absmax/_layer_51_input_layernorm":32.75,"hidden-states-absmax/_layer_16_gated_delta::in_proj":17.75,"param-mean/_layer_46_gated_delta::out_proj":3.067884244956076e-05,"hidden-states-rms/_layer_41_moe::router":0.07216878508256834,"update-absmax/_layer_23_gated_delta::out_norm":0.0078125,"grad-rms/moe::shared_experts":2.2269453235145132e-06,"hidden-states-absmax/attention::o_proj":696,"grad-rms/layer_05/moe::expert_fc1":1.1136857432437105e-08,"param-mean/_layer_09_gated_delta::out_proj":-1.720126783766318e-05,"param-l1/_layer_30_gated_delta::conv1d":0.038004860281944275,"hidden-states-mean/_layer_44_gated_delta::gate":-0.16601400566268035,"update-mean/_layer_55_gated_delta::A_log":0.0001527685672044754,"hidden-states-mean/_layer_41_mlp":-0.0029165128928736322,"hidden-states-rms/_layer_08_residual_attn":0.702983248022115,"update-mean/_layer_13_attention::linear_qkv":4.677217280146806e-09,"param-absmax/_layer_09_gated_delta::dt_bias":0.6963508725166321,"hidden-states-l1/_layer_26_gated_delta::key":0.059253859878046496,"param-absmax/_layer_41_gated_delta::dt_bias":0.6056122779846191,"update-mean/_layer_37_gated_delta::out_norm":-0.00018310546875,"hidden-states-l1/_layer_54_input_layernorm":0.2723856893045532,"hidden-states-absmax/_layer_06_moe::shared_experts":13.9375,"hidden-states-l1/_layer_56_gated_delta::value":0.08360151453036244,"update-absmax/_layer_05_gated_delta::out_norm":0.0029296875,"hidden-states-absmax/_layer_35_gated_delta::out_norm":80.5,"hidden-states-absmax/_layer_06_residual_attn":46.5,"hidden-states-l1/_layer_39_gated_delta::gate":0.9516434611849143,"update-absmax/_layer_23_gated_delta::dt_bias":0.0016681402921676636,"hidden-states-absmax/_layer_42_gated_delta::query":5.21875,"update-l1/_layer_37_gated_delta::out_norm":0.00067138671875,"grad-absmax/layer_41/gated_delta::out_norm":1.7031643437803723e-05,"hidden-states-l1/_layer_12_gated_delta::in_proj":0.8710828118107936,"grad-absmax/layer_20/moe::shared_expert_fc1":8.819306822260842e-05,"hidden-states-l1/_layer_08_moe::shared_experts":0.032853014958264225,"hidden-states-mean/_layer_40_gated_delta::out_norm":-0.0019899222578015634,"grad-rms/layer_26/moe::shared_experts":1.2991104938491393e-06,"param-absmax/_layer_22_attention::linear_qkv":0.65234375,"hidden-states-rms/_layer_09_moe::routed_experts":0.03441106762532799,"hidden-states-rms/_layer_52_gated_delta::beta":1.5306143839979425,"param-rms/_layer_53_gated_delta::out_norm":1.4647457649520015,"hidden-states-absmax/_layer_45_gated_delta::g":23.125,"update-l1/_layer_29_gated_delta::conv1d":0.0006359918043017387,"moe/layer_19/act_rms_min_to_median":0.29912709299903256,"hidden-states-absmax/_layer_25_gated_delta::query":6,"update-l1/_layer_43_gated_delta::out_norm":0.00054931640625,"update-l1/_layer_02_gated_delta::A_log":0.0005595450638793409,"hidden-states-absmax/_layer_47_moe::shared_experts":27.625,"grad-rms/layer_20/moe::router":9.326684498015602e-07,"hidden-states-l1/_layer_43_gated_delta::out_proj":0.38204409266043815,"hidden-states-l1/_layer_52_gated_delta::beta":1.1807228542235917,"hidden-states-absmax/_layer_46_gated_delta::core_attention":1.203125,"hidden-states-l1/_layer_30_gated_delta::g":5.006117463841136,"update-l1/_layer_28_gated_delta::out_proj":0.00013988098362460732,"param-l1/_layer_48_gated_delta::conv1d":0.026035666465759277,"hidden-states-l1/_layer_48_pre_mlp_layernorm":0.18989454983407852,"hidden-states-rms/_layer_19_input_layernorm":0.34021298149547863,"hidden-states-mean/_layer_40_moe::shared_experts":-0.0005487799716590311,"hidden-states-absmax/_layer_11_gated_delta::beta":23.5,"hidden-states-l1/_layer_42_gated_delta::out_proj":0.3531249884515987,"grad-absmax/layer_08/gated_delta::dt_bias":4.792467279912671e-06,"hidden-states-rms/_layer_05_moe::shared_experts":0.03965664255221743,"update-l1/_layer_48_gated_delta::A_log":0.0005442434921860695,"update-l1/_layer_10_gated_delta::A_log":0.0004098201170563698,"update-mean/_layer_26_gated_delta::conv1d":-8.944635737861972e-06,"grad-absmax/layer_17/moe::shared_expert_fc1":1.5501327652600594e-05,"hidden-states-absmax/_layer_37_residual_attn":502,"hidden-states-l1/_layer_41_residual_mlp":2.650820571281978,"hidden-states-absmax/_layer_54_moe::routed_experts":51.75,"hidden-states-mean/_layer_12_residual_mlp":-0.0005614014313177981,"update-absmax/_layer_39_gated_delta::out_norm":0.0078125,"update-rms/_layer_31_gated_delta::A_log":0.0007884202128974001,"hidden-states-rms/_layer_07_gated_delta::g":5.045485851053276,"moe/layer_49/act_rms_min":0.05692075192928314,"hidden-states-absmax/_layer_20_residual_attn":494,"update-rms/_layer_35_gated_delta::conv1d":0.0008221471561254414,"param-l1/_layer_31_gated_delta::A_log":1.258921504020691,"moe/global/act_min_to_median_lt_0_1_ratio":0.037037037037037035,"param-rms/_layer_06_gated_delta::A_log":1.447609637889838,"hidden-states-absmax/_layer_47_input_layernorm":47.5,"update-mean/_layer_02_gated_delta::dt_bias":-0.0001327277859672904,"hidden-states-l1/_layer_03_gated_delta::beta":1.3388507770122025,"hidden-states-mean/_layer_18_gated_delta::core_attention":4.3804355065298685e-05,"update-rms/_layer_03_gated_delta::out_norm":0.000702567409254232,"hidden-states-mean/_layer_08_gated_delta::key":0.013250579774448846,"update-l1/_layer_05_gated_delta::A_log":0.0004069376736879349,"hidden-states-absmax/_layer_43_moe::routed_experts":24.125,"hidden-states-absmax/_layer_42_gated_delta::out_proj":37.75,"hidden-states-mean/_layer_54_gated_delta::out_norm":-0.00747564896650898,"hidden-states-absmax/_layer_31_moe::routed_experts":27.125,"hidden-states-l1/_layer_07_moe::routed_experts":0.016492628071009444,"hidden-states-rms/_layer_34_gated_delta::value":0.11322152125526236,"hidden-states-l1/_layer_38_residual_attn":2.347002786040638,"hidden-states-mean/_layer_54_gated_delta::key":-0.009027966443057828,"hidden-states-rms/_layer_29_gated_delta::key":0.22224088524862845,"hidden-states-mean/_layer_32_gated_delta::out_norm":-0.0010443838626901939,"hidden-states-mean/_layer_05_gated_delta::gate":-0.27807119823350884,"grad-absmax/layer_38/moe::shared_expert_fc1":1.5003389307821635e-05,"hidden-states-rms/_layer_10_gated_delta::out_norm":0.05473306480646031,"param-l1/_layer_37_gated_delta::out_norm":1.19329833984375,"param-l1/_layer_45_gated_delta::A_log":1.021304965019226,"moe/layer_03/act_rms_min_to_median":0.21063225973325245,"param-absmax/_layer_52_gated_delta::conv1d":0.55859375,"grad-absmax/layer_01/mlp::linear_fc2":6.8831977841909975e-06,"hidden-states-absmax/_layer_10_moe::shared_experts":470,"update-l1/_layer_52_gated_delta::conv1d":0.0005945718730799854,"hidden-states-l1/_layer_25_gated_delta::key":0.15294750367969318,"hidden-states-rms/_layer_49_moe::router":0.07216878508256834,"hidden-states-absmax/_layer_52_residual_attn":2720,"param-absmax/_layer_53_gated_delta::out_proj":1.078125,"hidden-states-mean/_layer_11_input_layernorm":0.0006342391308235086,"hidden-states-mean/_layer_43_gated_delta::out_norm":-0.0018439007559970923,"param-mean/_layer_43_gated_delta::dt_bias":0.3516674041748047,"hidden-states-absmax/_layer_10_gated_delta::query":7.75,"grad-rms/layer_36/moe::expert_fc2":2.1458988613825374e-08,"hidden-states-absmax/_layer_27_gated_delta::key":8.875,"hidden-states-rms/_layer_52_input_layernorm":0.7633377930956609,"param-rms/_layer_27_gated_delta::conv1d":0.08187697867739817,"hidden-states-l1/_layer_08_mlp":0.055494914858927116,"moe/layer_26/act_rms_median":0.08511489629745483,"hidden-states-rms/_layer_43_moe::shared_experts":0.3193674524829276,"grad-absmax/layer_07/moe::shared_expert_fc2":1.878912598840543e-06,"hidden-states-mean/_layer_00_embeddings":-0.0005608510725763319,"grad-absmax/layer_04/moe::shared_expert_fc1":1.335139950242592e-05,"hidden-states-absmax/_layer_52_moe::shared_experts":78,"hidden-states-rms/_layer_05_gated_delta::core_attention":0.0020835787044639568,"hidden-states-rms/_layer_20_moe::router":0.07216878508256834,"grad-rms/layer_56/gated_delta::conv1d":0.0001410025530763937,"hidden-states-rms/_layer_36_residual_mlp":7.037886556540428,"hidden-states-rms/_layer_19_moe::shared_experts":0.11293986540467567,"hidden-states-mean/_layer_36_residual_mlp":-0.07495967447277004,"update-absmax/_layer_35_gated_delta::dt_bias":0.0018128007650375366,"update-mean/_layer_39_gated_delta::A_log":-0.000269551295787096,"update-l1/_layer_53_gated_delta::dt_bias":0.0005010399036109447,"grad-rms/layer_27/gated_delta::A_log":2.5595642068845417e-06,"param-rms/_layer_18_gated_delta::A_log":1.0210252244133073,"hidden-states-l1/_layer_46_moe::routed_experts":0.13543208576243396,"hidden-states-rms/_layer_49_attention::core_attention":0.5666770362618816,"param-mean/_layer_29_gated_delta::conv1d":0.00035729233059100807,"param-mean/_layer_10_gated_delta::out_proj":4.011216515209526e-05,"hidden-states-l1/final_layernorm":0.4667140205757642,"hidden-states-absmax/_layer_42_gated_delta::in_proj":25.125,"grad-rms/layer_14/gated_delta::dt_bias":2.0819676057168866e-06,"hidden-states-absmax/_layer_14_pre_mlp_layernorm":2.765625,"hidden-states-l1/_layer_39_moe::routed_experts":0.08509144383701114,"grad-rms/layer_21/moe::router":7.687173958978605e-07,"hidden-states-rms/_layer_37_gated_delta::in_proj":1.4122262813285036,"grad-absmax/layer_33/gated_delta::conv1d":0.00015601800987496972,"hidden-states-l1/_layer_18_gated_delta::beta":1.2055987364767475,"grad-rms/layer_14/gated_delta::conv1d":1.2092925033078833e-05,"hidden-states-mean/_layer_05_input_layernorm":0.0016232278037566308,"param-rms/_layer_26_gated_delta::dt_bias":0.17564459893297324,"hidden-states-mean/_layer_44_moe::routed_experts":-0.0002853404037170725,"param-absmax/_layer_18_gated_delta::dt_bias":0.37254512310028076,"grad-rms/layer_14/moe::shared_experts":1.1692461911930669e-06,"grad-rms/layer_13/moe::expert_fc2":2.3392987112501896e-08,"hidden-states-rms/_layer_44_residual_attn":8.181949531836226,"grad-rms/layer_42/moe::shared_experts":2.2551646823323927e-06,"update-rms/_layer_46_gated_delta::A_log":0.0005307038247817949,"param-l1/_layer_49_attention::linear_qkv":0.028086896985769272,"param-rms/_layer_22_attention::linear_qkv":0.03797035683118388,"grad-absmax/lm_head":0.004123541060835123,"hidden-states-mean/_layer_06_moe::routed_experts":-1.64311404305526e-05,"hidden-states-absmax/_layer_17_input_layernorm":17.5,"hidden-states-rms/_layer_08_moe::shared_experts":0.04552392669455859,"moe/layer_51/act_rms_median":0.1132606714963913,"moe/layer_06/act_rms_min_to_median":0.24276098913963765,"hidden-states-absmax/_layer_33_mlp":28.875,"hidden-states-absmax/_layer_36_gated_delta::query":4.25,"moe/layer_34/act_rms_min_to_median":0.5016848534706666,"hidden-states-rms/_layer_37_gated_delta::out_norm":0.1886866504428473,"hidden-states-absmax/_layer_09_pre_mlp_layernorm":2.0625,"hidden-states-mean/_layer_38_gated_delta::key":0.015296440532379213,"hidden-states-rms/_layer_11_moe::shared_experts":0.06460438689932642,"param-absmax/_layer_05_gated_delta::out_proj":0.73828125,"hidden-states-rms/_layer_32_gated_delta::query":1.4099902105698556,"hidden-states-rms/_layer_27_gated_delta::beta":2.0911799463529284,"param-mean/_layer_51_gated_delta::dt_bias":0.45040082931518555,"hidden-states-absmax/_layer_36_moe::router":0.9999668002128601,"update-mean/_layer_36_gated_delta::A_log":-0.0001556258648633957,"hidden-states-absmax/_layer_16_gated_delta::core_attention":0.09033203125,"grad-absmax/layer_55/moe::router":0.00016218451492022723,"update-l1/_layer_42_gated_delta::dt_bias":0.0004183722485322505,"grad-absmax/layer_47/moe::shared_expert_fc2":9.153418432106264e-06,"hidden-states-rms/_layer_07_gated_delta::query":0.12087242554609637,"grad-absmax/layer_36/moe::expert_fc1":9.023005986819044e-06,"hidden-states-mean/_layer_31_mlp":-0.0017110844785522793,"update-l1/_layer_11_gated_delta::out_norm":0.00067901611328125,"hidden-states-rms/_layer_31_gated_delta::out_proj":0.35088949391868407,"hidden-states-l1/_layer_06_gated_delta::core_attention":0.0012947663578485081,"grad-rms/layer_51/moe::shared_expert_fc2":3.8847912160735516e-07,"param-absmax/_layer_39_gated_delta::out_proj":0.66015625,"param-mean/_layer_14_gated_delta::conv1d":-0.0018183272331953049,"hidden-states-l1/_layer_55_residual_mlp":6.750384658620913,"hidden-states-rms/_layer_22_attention::k_layernorm":3.5220037009394347,"update-rms/_layer_38_gated_delta::A_log":0.0005691347625883625,"hidden-states-l1/_layer_41_moe::router":0.010099385297200726,"hidden-states-mean/_layer_28_pre_mlp_layernorm":-0.00036182986461950716,"hidden-states-l1/_layer_11_gated_delta::key":0.07489868401273299,"moe/layer_48/act_rms_median":0.28305715322494507,"param-l1/_layer_02_gated_delta::A_log":1.1245478391647339,"hidden-states-absmax/_layer_06_gated_delta::beta":15.5,"moe/layer_47/act_rms_max":0.6404206156730652,"hidden-states-absmax/_layer_35_moe::shared_experts":30.75,"hidden-states-absmax/_layer_42_residual_attn":502,"grad-rms/layer_05/moe::shared_experts":4.087934261802343e-07,"update-rms/_layer_14_gated_delta::out_norm":0.0013593174713940482,"hidden-states-l1/_layer_26_gated_delta::gate":0.7655415851523382,"hidden-states-absmax/_layer_50_moe::routed_experts":123.5,"hidden-states-rms/_layer_37_gated_delta::out_proj":0.36341059969249223,"moe/layer_18/act_rms_min_to_median":0.37612878549787176,"param-mean/_layer_41_gated_delta::out_proj":3.318426024634391e-05,"param-absmax/_layer_48_gated_delta::conv1d":0.5,"hidden-states-rms/_layer_20_gated_delta::core_attention":0.002580954647226136,"param-mean/_layer_55_gated_delta::out_norm":1.368896484375,"hidden-states-l1/_layer_12_gated_delta::beta":2.098337662021835,"hidden-states-rms/_layer_35_gated_delta::key":0.1863331759170258,"param-l1/_layer_50_attention::linear_qkv":0.028719648718833923,"hidden-states-rms/_layer_30_gated_delta::value":0.05350228543989444,"hidden-states-l1/_layer_40_gated_delta::gate":1.3900584447453552,"grad-rms/layer_38/moe::shared_expert_fc2":1.1484189736139534e-07,"hidden-states-absmax/_layer_36_gated_delta::value":9.5,"param-absmax/_layer_45_gated_delta::out_norm":1.46875,"update-l1/attention::linear_qkv":0.00013327008387672583,"hidden-states-l1/moe::routed_experts":0.06853728076239526,"update-absmax/_layer_45_gated_delta::out_proj":0.001953125,"hidden-states-rms/_layer_03_gated_delta::beta":1.7844557837149997,"hidden-states-l1/_layer_43_gated_delta::query":0.13451160603900028,"param-rms/_layer_40_gated_delta::conv1d":0.0691917828239669,"param-l1/_layer_03_gated_delta::A_log":1.300605297088623,"grad-rms/layer_54/gated_delta::conv1d":0.00031934721721588817,"param-rms/_layer_05_gated_delta::A_log":1.4388296363459325,"param-l1/_layer_12_gated_delta::out_proj":0.03317531570792198,"hidden-states-l1/_layer_39_residual_mlp":2.4730017749073236,"hidden-states-mean/_layer_10_residual_attn":0.0005984885050975984,"hidden-states-rms/_layer_46_gated_delta::core_attention":0.011390539158697515,"hidden-states-l1/_layer_01_gated_delta::g":7.817777611944391,"hidden-states-absmax/_layer_07_moe::routed_experts":8.125,"hidden-states-rms/_layer_44_moe::shared_experts":0.3774204448602882,"hidden-states-absmax/_layer_21_residual_mlp":498,"grad-absmax/layer_23/moe::expert_fc1":1.6593721738900058e-05,"grad-rms/layer_24/attention::o_proj":1.1066897626501122e-06,"grad-absmax/layer_52/moe::expert_fc1":2.534809391363524e-05,"param-absmax/_layer_50_attention::o_proj":1.9453125,"param-mean/_layer_36_gated_delta::dt_bias":0.5432618260383606,"update-mean/_layer_13_attention::o_proj":-1.8591083517094376e-07,"hidden-states-l1/_layer_28_gated_delta::out_norm":0.0773763078303704,"hidden-states-rms/_layer_12_pre_mlp_layernorm":0.08864176636941447,"hidden-states-l1/_layer_06_gated_delta::out_proj":0.05918971409675051,"hidden-states-rms/_layer_13_residual_attn":1.0598327099873335,"update-l1/_layer_09_gated_delta::conv1d":0.0006576827145181596,"grad-rms/layer_06/gated_delta::out_norm":7.992830628844857e-05,"param-rms/_layer_06_gated_delta::out_norm":0.27954188184312434,"hidden-states-mean/_layer_11_gated_delta::out_norm":-0.0005716650729748694,"hidden-states-rms/_layer_31_moe::shared_experts":0.1876097112425002,"hidden-states-rms/_layer_31_gated_delta::out_norm":0.16459150254071991,"hidden-states-absmax/_layer_40_residual_mlp":502,"param-mean/_layer_04_gated_delta::A_log":0.8990836143493652,"param-absmax/_layer_07_gated_delta::conv1d":0.77734375,"hidden-states-rms/_layer_34_gated_delta::g":6.591033334700648,"grad-rms/layer_28/moe::shared_expert_fc2":1.4369780397688277e-07,"hidden-states-rms/_layer_30_gated_delta::gate":1.0152963564829893,"hidden-states-mean/_layer_05_gated_delta::g":-3.2857405287910137,"param-rms/_layer_05_gated_delta::out_norm":0.2405012013535225,"hidden-states-absmax/gated_delta::gate":113,"param-l1/_layer_27_gated_delta::out_norm":0.735137939453125,"grad-rms/layer_12/moe::shared_expert_fc1":5.528076587174714e-07,"hidden-states-mean/_layer_16_gated_delta::query":0.007792427202156449,"update-rms/_layer_28_gated_delta::dt_bias":0.0005461234260717648,"param-absmax/_layer_16_gated_delta::out_norm":0.8515625,"grad-absmax/layer_45/moe::router":0.00012067778880009428,"grad-rms/layer_38/moe::expert_fc1":2.1111325581486352e-08,"update-l1/_layer_23_gated_delta::conv1d":0.0005890045431442559,"hidden-states-mean/_layer_34_pre_mlp_layernorm":0.00026685817554392523,"update-mean/_layer_12_gated_delta::dt_bias":7.003603968769312e-05,"hidden-states-rms/_layer_17_moe::routed_experts":0.05447153664403124,"hidden-states-absmax/_layer_10_gated_delta::value":6.25,"grad-rms/layer_06/moe::expert_fc2":1.9162171721824155e-08,"hidden-states-l1/_layer_15_gated_delta::query":0.049521213712836944,"hidden-states-absmax/_layer_38_gated_delta::key":3.71875,"hidden-states-mean/_layer_46_gated_delta::out_proj":-0.0009398904468919869,"grad-rms/layer_27/moe::shared_expert_fc1":2.669935213893897e-07,"param-absmax/_layer_16_gated_delta::dt_bias":0.36235854029655457,"grad-rms/layer_21/attention::o_proj":1.4935775694142915e-06,"hidden-states-rms/_layer_10_gated_delta::out_proj":0.11586369687076349,"update-l1/_layer_42_gated_delta::conv1d":0.0006124519277364016,"param-absmax/_layer_41_gated_delta::out_proj":1.015625,"moe/layer_30/act_rms_min":0.050391748547554016,"update-absmax/_layer_35_gated_delta::conv1d":0.00390625,"update-l1/lm_head":0.00019701816199813038,"update-rms/_layer_11_gated_delta::dt_bias":0.0006266293199548141,"param-l1/_layer_32_gated_delta::conv1d":0.054356545209884644,"hidden-states-rms/_layer_03_gated_delta::key":0.06711345945877496,"param-mean/_layer_18_gated_delta::out_proj":-2.2402118702302687e-05,"param-l1/_layer_17_gated_delta::dt_bias":0.16669327020645142,"hidden-states-mean/_layer_24_moe::routed_experts":-0.0002117405289862914,"hidden-states-absmax/_layer_40_gated_delta::beta":11.4375,"update-l1/_layer_28_gated_delta::A_log":0.00042870501056313515,"hidden-states-mean/_layer_51_moe::routed_experts":-5.156937978694353e-05,"hidden-states-rms/_layer_53_gated_delta::in_proj":2.383076176636516,"param-l1/_layer_36_gated_delta::out_proj":0.03145753964781761,"update-mean/lm_head":-1.2675708376264083e-06,"hidden-states-absmax/_layer_47_attention::q_layernorm":36,"hidden-states-rms/_layer_47_mlp":0.867339026626166,"grad-rms/layer_07/moe::expert_fc1":1.3427816712569859e-08,"param-rms/_layer_36_gated_delta::A_log":1.6343874841508665,"hidden-states-l1/_layer_19_moe::routed_experts":0.04499686888573549,"hidden-states-mean/_layer_50_pre_mlp_layernorm":-0.0009409842271786007,"hidden-states-mean/_layer_27_gated_delta::out_norm":-0.001322701265388878,"update-l1/_layer_25_gated_delta::out_proj":0.00014684606867376715,"hidden-states-l1/_layer_01_pre_mlp_layernorm":0.05217589314888634,"grad-absmax/layer_14/gated_delta::dt_bias":5.985495135973906e-06,"hidden-states-rms/_layer_28_input_layernorm":0.43944401277866113,"param-absmax/_layer_32_gated_delta::A_log":1.8807109594345093,"hidden-states-rms/_layer_55_gated_delta::key":0.14650662172954335,"hidden-states-absmax/_layer_35_gated_delta::in_proj":30.75,"param-rms/_layer_08_gated_delta::out_proj":0.048144376398576466,"moe/layer_53/act_rms_median":0.10653340071439743,"update-mean/_layer_43_gated_delta::out_norm":-6.103515625e-05,"param-l1/_layer_34_gated_delta::out_proj":0.034114543348550797,"hidden-states-mean/_layer_05_moe::routed_experts":3.963164121442829e-06,"hidden-states-absmax/_layer_42_gated_delta::value":5.65625,"hidden-states-absmax/_layer_17_gated_delta::gate":11.25,"grad-absmax/layer_49/moe::expert_fc2":1.3557923011831008e-05,"hidden-states-mean/_layer_11_moe::shared_experts":0.00013899000097598893,"hidden-states-l1/_layer_06_pre_mlp_layernorm":0.05134229974058863,"update-mean/_layer_30_gated_delta::out_proj":-9.908934117675017e-08,"hidden-states-l1/_layer_17_pre_mlp_layernorm":0.08771098705661143,"hidden-states-mean/_layer_24_attention::k_layernorm":-0.05173693469011005,"hidden-states-l1/_layer_19_gated_delta::in_proj":0.7372935012340311,"update-mean/_layer_06_gated_delta::conv1d":-2.3828953999327496e-05,"update-mean/_layer_50_attention::linear_qkv":-7.661003564862767e-08,"grad-absmax/layer_29/moe::router":8.818641072139144e-05,"hidden-states-mean/_layer_43_moe::shared_experts":-0.0007043066692084573,"hidden-states-l1/_layer_05_moe::routed_experts":0.011866760346569085,"hidden-states-l1/_layer_36_residual_attn":2.1678171357890528,"hidden-states-mean/_layer_30_gated_delta::key":0.0070532266076775325,"grad-absmax/layer_52/moe::shared_expert_fc1":0.00023853527090977877,"hidden-states-absmax/_layer_42_input_layernorm":25.125,"grad-absmax/layer_20/moe::expert_fc2":5.050615527579794e-06,"param-mean/_layer_02_gated_delta::conv1d":-0.0011698976159095764,"hidden-states-l1/_layer_18_residual_mlp":0.589779685709439,"hidden-states-mean/_layer_19_pre_mlp_layernorm":-0.0005033043580343554,"hidden-states-absmax/_layer_25_gated_delta::value":2.046875,"grad-absmax/layer_27/moe::expert_fc2":1.0174376257054973e-06,"hidden-states-mean/_layer_12_residual_attn":-0.0005159641183385692,"param-absmax/_layer_49_attention::linear_qkv":0.578125,"param-l1/_layer_02_gated_delta::out_proj":0.034811317920684814,"hidden-states-l1/_layer_24_attention::core_attention":0.32321546354478164,"hidden-states-rms/_layer_10_gated_delta::beta":2.3025355052195957,"grad-absmax/layer_50/attention::linear_qkv":0.0004737941490020603,"param-l1/_layer_36_gated_delta::dt_bias":0.5432618260383606,"hidden-states-l1/_layer_18_gated_delta::g":4.82746610968901,"param-mean/_layer_33_gated_delta::out_proj":3.901924719684757e-05,"update-absmax/_layer_31_gated_delta::A_log":0.0016388893127441406,"moe/layer_14/act_rms_max":0.08894351869821548,"hidden-states-absmax/_layer_30_gated_delta::g":15.125,"param-mean/_layer_44_gated_delta::out_norm":1.04547119140625,"hidden-states-absmax/_layer_56_residual_attn":2816,"grad-rms/layer_11/gated_delta::conv1d":1.2191321231122334e-05,"grad-rms/layer_54/moe::expert_fc1":1.3818277678920269e-08,"hidden-states-absmax/_layer_14_moe::routed_experts":11.3125,"hidden-states-absmax/_layer_02_gated_delta::out_norm":5.84375,"grad-absmax/layer_18/gated_delta::conv1d":0.0005441958201117814,"hidden-states-l1/_layer_47_pre_mlp_layernorm":0.19893012332083265,"hidden-states-l1/_layer_25_gated_delta::out_proj":0.23270773681494436,"hidden-states-mean/gated_delta::gate":-0.25456858427575807,"param-l1/_layer_46_gated_delta::A_log":1.1678706407546997,"param-l1/_layer_18_gated_delta::conv1d":0.03698455169796944,"hidden-states-absmax/_layer_10_mlp":470,"param-l1/_layer_28_gated_delta::out_proj":0.03346512094140053,"update-absmax/_layer_17_gated_delta::out_norm":0.00390625,"param-absmax/_layer_11_gated_delta::out_norm":0.83984375,"hidden-states-absmax/_layer_52_gated_delta::gate":41,"grad-rms/layer_35/gated_delta::A_log":2.364519439630356e-06,"param-rms/_layer_20_gated_delta::dt_bias":0.33132852352363923,"hidden-states-rms/_layer_06_residual_mlp":0.6525384236315347,"hidden-states-l1/_layer_37_mlp":0.20820371308370986,"hidden-states-l1/_layer_55_moe::shared_experts":0.13111375345155638,"hidden-states-mean/_layer_41_residual_attn":-0.08856506956399317,"hidden-states-rms/_layer_27_gated_delta::value":0.037558669220174395,"hidden-states-mean/_layer_52_moe::router":0.010327688507666351,"grad-absmax/moe::expert_fc2":3.0134706321405247e-05,"hidden-states-rms/_layer_11_input_layernorm":0.36903640425281153,"param-rms/_layer_08_gated_delta::A_log":1.235266351588128,"hidden-states-mean/_layer_41_gated_delta::value":-0.0018348481830773571,"hidden-states-absmax/_layer_31_gated_delta::out_norm":55.25,"grad-absmax/layer_11/gated_delta::conv1d":0.0003252271853853017,"update-rms/_layer_36_gated_delta::A_log":0.000523384879805032,"grad-absmax/moe::expert_fc1":2.942052742582746e-05,"hidden-states-absmax/_layer_14_gated_delta::value":6,"param-absmax/_layer_01_gated_delta::conv1d":0.376953125,"update-absmax/_layer_04_gated_delta::dt_bias":0.0013508200645446777,"param-l1/_layer_03_gated_delta::out_proj":0.03578478470444679,"hidden-states-l1/_layer_20_gated_delta::gate":0.9432187012694718,"update-absmax/_layer_29_gated_delta::dt_bias":0.002105146646499634,"hidden-states-rms/_layer_11_gated_delta::beta":2.7144550153471503,"hidden-states-l1/_layer_19_gated_delta::g":6.6954568261452705,"hidden-states-absmax/_layer_43_gated_delta::g":16.25,"hidden-states-l1/_layer_10_residual_mlp":0.3304165995617628,"hidden-states-absmax/_layer_25_gated_delta::g":17.375,"update-absmax/_layer_54_gated_delta::A_log":0.0011883378028869629,"grad-absmax/layer_26/moe::shared_experts":1.1228838047827594e-05,"update-absmax/_layer_38_gated_delta::out_proj":0.001953125,"hidden-states-rms/_layer_12_gated_delta::in_proj":1.4138693190992198,"hidden-states-mean/_layer_25_gated_delta::out_proj":0.0001757354102974129,"hidden-states-absmax/_layer_02_pre_mlp_layernorm":4.15625,"hidden-states-absmax/_layer_27_moe::routed_experts":13.375,"hidden-states-mean/_layer_09_gated_delta::g":-5.179069659599025,"hidden-states-rms/_layer_56_gated_delta::query":0.16992542574437933,"update-rms/_layer_07_gated_delta::dt_bias":0.0007716895663513805,"update-l1/_layer_07_gated_delta::A_log":0.0005827771965414286,"param-absmax/_layer_07_gated_delta::A_log":1.939001202583313,"grad-rms/layer_49/moe::expert_fc2":2.137220870134178e-08,"hidden-states-mean/_layer_48_gated_delta::beta":1.0419712116683975,"param-mean/_layer_00_embedding":0.0014617833076044917,"update-absmax/_layer_36_gated_delta::out_proj":0.00390625,"hidden-states-l1/_layer_11_gated_delta::query":0.057657628674224456,"hidden-states-rms/_layer_03_gated_delta::value":0.06331177753004377,"param-mean/_layer_53_gated_delta::out_norm":1.4378662109375,"hidden-states-mean/_layer_26_moe::shared_experts":-0.0005801312036409919,"param-l1/_layer_06_gated_delta::out_proj":0.0349026620388031,"hidden-states-l1/_layer_56_gated_delta::out_norm":0.26472330268899236,"grad-rms/layer_50/moe::router":1.397246444341191e-06,"param-l1/_layer_23_gated_delta::conv1d":0.038421630859375,"hidden-states-rms/_layer_31_gated_delta::value":0.09305159434056168,"grad-absmax/layer_13/attention::o_proj":5.995315223117359e-05,"grad-rms/layer_36/gated_delta::conv1d":0.00011463106025520422,"grad-rms/layer_10/gated_delta::A_log":3.010992673900144e-06,"hidden-states-absmax/_layer_02_gated_delta::out_proj":4.0625,"update-rms/_layer_17_gated_delta::out_proj":0.00020238578740059608,"hidden-states-l1/_layer_44_gated_delta::core_attention":0.0020921677516749803,"grad-absmax/layer_22/moe::router":6.948460941202939e-05,"hidden-states-absmax/_layer_17_moe::router":0.999830424785614,"grad-absmax/layer_02/mlp::linear_fc2":8.892357072909363e-06,"hidden-states-l1/_layer_40_gated_delta::core_attention":0.0015255226243910342,"grad-rms/layer_08/moe::expert_fc2":1.970486795128262e-08,"update-rms/_layer_05_gated_delta::A_log":0.0005131703075711902,"grad-absmax/layer_46/moe::expert_fc2":3.675152584037278e-06,"param-absmax/_layer_25_gated_delta::out_norm":1.75,"update-mean/_layer_28_gated_delta::dt_bias":-2.136339753633365e-05,"grad-rms/layer_23/moe::router":7.746963058565833e-07,"param-absmax/_layer_29_gated_delta::conv1d":0.5859375,"hidden-states-absmax/_layer_18_moe::routed_experts":6.90625,"hidden-states-mean/_layer_46_gated_delta::gate":-0.0009764061542041486,"hidden-states-l1/_layer_48_moe::routed_experts":0.15568905626119664,"hidden-states-l1/_layer_18_moe::shared_experts":0.061504643291788494,"update-l1/_layer_09_gated_delta::out_proj":0.00015124992933124304,"hidden-states-absmax/_layer_14_gated_delta::query":7.40625,"grad-rms/layer_46/gated_delta::A_log":8.369092456424917e-06,"param-l1/_layer_42_gated_delta::conv1d":0.03347126767039299,"hidden-states-rms/_layer_43_pre_mlp_layernorm":0.23387658649027226,"grad-rms/layer_32/moe::expert_fc1":1.97073542713703e-08,"hidden-states-rms/_layer_52_gated_delta::query":0.3837213078440138,"update-l1/_layer_28_gated_delta::dt_bias":0.00042607777868397534,"hidden-states-mean/_layer_42_gated_delta::gate":0.2510138453261668,"hidden-states-rms/_layer_41_mlp":0.48317936825306357,"hidden-states-absmax/_layer_10_residual_attn":66,"grad-absmax/layer_51/gated_delta::A_log":3.596118767745793e-05,"grad-absmax/layer_33/gated_delta::A_log":2.3086251985660056e-06,"grad-rms/layer_26/moe::router":5.509821946174851e-07,"hidden-states-rms/_layer_16_gated_delta::gate":1.1686528663127,"hidden-states-absmax/_layer_35_gated_delta::out_proj":32.5,"update-l1/_layer_08_gated_delta::conv1d":0.0006841467693448067,"moe/layer_18/act_rms_median":0.06246159225702286,"param-rms/_layer_33_gated_delta::conv1d":0.07840183503012173,"hidden-states-mean/_layer_20_moe::routed_experts":-3.364276978649163e-05,"update-l1/_layer_18_gated_delta::A_log":0.0004502632364165038,"hidden-states-rms/_layer_17_residual_attn":1.3146381549843276,"update-mean/mlp::linear_fc2":-4.179874313336995e-08,"param-rms/_layer_51_gated_delta::dt_bias":0.4710371960907666,"hidden-states-l1/_layer_20_gated_delta::core_attention":0.0009425848798368728,"hidden-states-l1/_layer_52_gated_delta::out_norm":0.20398537266080052,"hidden-states-l1/_layer_44_residual_attn":2.8981603262615594,"grad-rms/layer_06/gated_delta::dt_bias":4.1853357126238474e-05,"grad-rms/layer_53/moe::router":1.4206455723120143e-06,"update-rms/_layer_33_gated_delta::conv1d":0.0008369186296499751,"hidden-states-mean/_layer_21_pre_mlp_layernorm":-0.00034482961673121054,"hidden-states-absmax/_layer_45_mlp":73,"hidden-states-mean/_layer_35_gated_delta::value":0.009673317184843681,"hidden-states-rms/_layer_34_residual_mlp":6.183970419919875,"param-l1/_layer_09_gated_delta::dt_bias":0.24422478675842285,"hidden-states-rms/_layer_04_moe::router":0.07216878508256834,"param-l1/_layer_57_lm_head":0.039115045219659805,"hidden-states-l1/_layer_03_moe::routed_experts":0.008653178012037332,"hidden-states-rms/_layer_28_residual_attn":3.4381023194964104,"grad-rms/layer_17/moe::expert_fc2":2.253249578221996e-08,"param-absmax/_layer_03_gated_delta::A_log":2.204395055770874,"update-absmax/_layer_07_gated_delta::dt_bias":0.0017226338386535645,"update-mean/_layer_01_mlp::linear_fc2":2.6669665587064628e-08,"hidden-states-absmax/_layer_21_attention::k_layernorm":25.625,"grad-absmax/layer_51/moe::expert_fc1":2.0451545424293727e-05,"update-rms/_layer_32_gated_delta::out_norm":0.0010429692071920813,"hidden-states-mean/_layer_42_moe::routed_experts":-0.00023371153873353952,"hidden-states-rms/_layer_05_gated_delta::key":0.10066092250228197,"hidden-states-mean/_layer_07_gated_delta::gate":-0.39742766588064027,"hidden-states-mean/gated_delta::core_attention":0.00043821957442820064,"hidden-states-l1/_layer_03_residual_mlp":0.1959091360460682,"hidden-states-rms/_layer_48_moe::routed_experts":0.30136054411332347,"update-l1/_layer_54_gated_delta::out_norm":0.00048828125,"hidden-states-mean/_layer_16_gated_delta::g":-5.9368693753325195,"update-mean/_layer_25_gated_delta::out_norm":-0.000152587890625,"hidden-states-l1/_layer_08_gated_delta::gate":0.7701332295099771,"hidden-states-absmax/_layer_44_pre_mlp_layernorm":6.15625,"param-mean/_layer_12_gated_delta::out_norm":0.518310546875,"param-absmax/_layer_44_gated_delta::A_log":1.7985508441925049,"hidden-states-absmax/_layer_39_residual_attn":502,"param-l1/_layer_38_gated_delta::out_norm":1.02032470703125,"hidden-states-rms/_layer_28_gated_delta::beta":3.15800461244641,"update-absmax/_layer_48_gated_delta::out_norm":0.0078125,"update-absmax/_layer_05_gated_delta::dt_bias":0.0011102259159088135,"hidden-states-rms/_layer_05_gated_delta::query":0.11151076106011897,"hidden-states-rms/_layer_05_gated_delta::gate":1.0535489250674337,"grad-rms/layer_30/gated_delta::out_proj":3.251885626757701e-07,"update-rms/_layer_45_gated_delta::dt_bias":0.0006020787639984866,"hidden-states-l1/_layer_34_residual_attn":1.9738855330807605,"hidden-states-l1/_layer_21_residual_mlp":0.7919366707726324,"hidden-states-absmax/_layer_04_gated_delta::beta":11.625,"hidden-states-mean/_layer_37_gated_delta::value":0.00011038104864188864,"hidden-states-l1/_layer_54_gated_delta::core_attention":0.0033700567962296956,"hidden-states-absmax/_layer_04_gated_delta::in_proj":19.5,"hidden-states-rms/_layer_28_moe::routed_experts":0.11074300580100298,"hidden-states-absmax/_layer_25_gated_delta::gate":16.125,"hidden-states-absmax/_layer_15_gated_delta::gate":13.125,"hidden-states-mean/_layer_17_gated_delta::key":0.015089566755541177,"update-rms/_layer_10_gated_delta::conv1d":0.0008321144512569691,"update-absmax/_layer_49_attention::linear_qkv":0.00390625,"update-absmax/_layer_38_gated_delta::conv1d":0.0029296875,"hidden-states-rms/_layer_09_gated_delta::g":6.2079042577030386,"hidden-states-l1/_layer_27_gated_delta::out_norm":0.06942505472005066,"update-l1/_layer_40_gated_delta::out_norm":0.0006103515625,"hidden-states-l1/_layer_07_gated_delta::value":0.03200408825327018,"grad-absmax/layer_26/moe::shared_expert_fc1":1.4013947293278761e-05,"hidden-states-rms/_layer_31_gated_delta::gate":1.5101832703919063,"hidden-states-l1/_layer_39_gated_delta::out_proj":0.32471750077217826,"update-rms/_layer_07_gated_delta::A_log":0.0007681374872807322,"hidden-states-absmax/_layer_23_gated_delta::value":6.0625,"hidden-states-mean/_layer_31_pre_mlp_layernorm":-4.588974299154906e-05,"grad-absmax/layer_41/moe::shared_expert_fc1":1.7679891243460588e-05,"update-l1/_layer_15_gated_delta::dt_bias":0.0005393221508711576,"grad-absmax/layer_54/moe::shared_experts":6.593163561774418e-06,"hidden-states-rms/_layer_39_gated_delta::out_proj":0.49294715785659304,"lm-global-samples":32,"hidden-states-mean/_layer_21_moe::router":0.010260928190498064,"hidden-states-mean/_layer_06_gated_delta::out_proj":-0.0003727085867027744,"param-rms/_layer_41_gated_delta::conv1d":0.06702391440371608,"grad-rms/layer_21/attention::linear_qkv":3.0212623286530217e-06,"grad-absmax/layer_46/gated_delta::dt_bias":4.5216744183562696e-05,"hidden-states-mean/_layer_39_gated_delta::gate":-0.3962998266577514,"grad-absmax/layer_34/gated_delta::A_log":1.4474772797257174e-05,"grad-absmax/layer_32/moe::expert_fc1":3.3045409963960992e-06,"grad-rms/layer_27/gated_delta::dt_bias":2.531118193770142e-06,"param-mean/_layer_46_gated_delta::A_log":1.0951699018478394,"param-rms/_layer_13_attention::o_proj":0.03529291445068759,"hidden-states-mean/_layer_04_input_layernorm":0.002925552543691104,"grad-rms/layer_31/moe::router":4.275021754817853e-07,"moe/layer_47/act_rms_median":0.23227538168430328,"hidden-states-mean/_layer_29_gated_delta::core_attention":9.668719273848521e-05,"hidden-states-mean/_layer_32_gated_delta::query":0.12338034789327953,"update-l1/_layer_19_gated_delta::out_proj":0.00013898582255933434,"param-l1/_layer_04_gated_delta::A_log":1.0178366899490356,"grad-rms/layer_04/moe::router":4.821916821481345e-07,"hidden-states-rms/_layer_16_moe::router":0.07216878508256834,"hidden-states-rms/_layer_50_residual_mlp":11.246323718533935,"hidden-states-absmax/_layer_39_gated_delta::g":16.375,"update-mean/_layer_20_gated_delta::out_norm":-9.1552734375e-05,"update-mean/_layer_47_attention::o_proj":2.1427868546197715e-07,"hidden-states-absmax/_layer_53_moe::routed_experts":266,"update-mean/_layer_20_gated_delta::conv1d":-4.185159923508763e-06,"grad-rms/layer_45/gated_delta::conv1d":3.361985093085651e-05,"update-l1/_layer_18_gated_delta::out_norm":0.000518798828125,"hidden-states-mean/_layer_55_gated_delta::out_proj":0.011118121839239816,"update-absmax/_layer_26_gated_delta::out_norm":0.0078125,"hidden-states-rms/_layer_37_gated_delta::key":0.11339384536599552,"hidden-states-l1/_layer_54_gated_delta::out_norm":0.22211229468730626,"param-mean/_layer_27_gated_delta::out_norm":0.735137939453125,"hidden-states-rms/_layer_27_moe::routed_experts":0.10554917538672856,"update-absmax/_layer_41_gated_delta::A_log":0.0013897567987442017,"hidden-states-mean/_layer_23_gated_delta::query":0.026545483058152564,"update-rms/_layer_07_gated_delta::out_norm":0.0009296598029619029,"grad-rms/layer_51/moe::shared_expert_fc1":5.585494619061056e-07,"update-absmax/_layer_46_gated_delta::A_log":0.0011606216430664062,"hidden-states-l1/_layer_29_pre_mlp_layernorm":0.13662412901212323,"hidden-states-mean/_layer_36_moe::router":0.010103617227652427,"hidden-states-l1/_layer_23_residual_mlp":0.9765033693913662,"hidden-states-absmax/_layer_01_mlp":13.5625,"hidden-states-mean/_layer_36_gated_delta::out_proj":-0.013684710256422788,"grad-rms/layer_51/gated_delta::out_proj":8.601753993458702e-07,"hidden-states-absmax/_layer_17_moe::shared_experts":5.125,"hidden-states-l1/_layer_45_moe::routed_experts":0.14500034633661263,"hidden-states-l1/_layer_32_input_layernorm":0.35281948480838354,"hidden-states-l1/_layer_46_moe::router":0.010210930440573762,"param-l1/_layer_05_gated_delta::A_log":1.3804713487625122,"grad-rms/layer_10/moe::router":2.935603935273368e-07,"param-mean/_layer_56_gated_delta::out_norm":1.3885498046875,"update-mean/_layer_10_gated_delta::A_log":-6.45732507109642e-05,"hidden-states-absmax/_layer_07_gated_delta::beta":12.8125,"hidden-states-rms/_layer_45_gated_delta::value":0.15575540794469572,"moe/layer_50/act_rms_min":0.03958677127957344,"hidden-states-absmax/_layer_30_gated_delta::beta":7.9375,"hidden-states-l1/_layer_27_moe::routed_experts":0.05754074436752293,"update-l1/_layer_52_gated_delta::A_log":0.0005603707395493984,"update-absmax/_layer_05_gated_delta::conv1d":0.003448486328125,"grad-rms/layer_33/gated_delta::dt_bias":1.0732227812364563e-06,"param-mean/_layer_42_gated_delta::conv1d":-0.0006921529420651495,"param-rms/_layer_07_gated_delta::out_norm":0.2377170856819248,"param-l1/_layer_15_gated_delta::out_norm":0.504150390625,"param-l1/_layer_02_gated_delta::conv1d":0.03381915017962456,"hidden-states-l1/_layer_49_residual_attn":4.055998808860637,"update-mean/_layer_19_gated_delta::out_proj":-2.469578248565085e-07,"moe/layer_04/act_rms_min_to_median":0.23170130235761008,"hidden-states-absmax/_layer_15_gated_delta::in_proj":22.875,"hidden-states-absmax/_layer_19_gated_delta::in_proj":18.75,"grad-rms/layer_45/moe::shared_experts":2.954556834259827e-06,"hidden-states-rms/_layer_15_gated_delta::query":0.09803508344826245,"param-mean/_layer_03_gated_delta::A_log":1.0645235776901245,"hidden-states-l1/_layer_10_gated_delta::query":0.04558636046485429,"param-mean/_layer_30_gated_delta::dt_bias":0.04775010049343109,"hidden-states-l1/_layer_04_moe::routed_experts":0.009908612094276147,"grad-rms/layer_26/gated_delta::out_norm":8.822827674327017e-06,"update-absmax/_layer_53_gated_delta::out_proj":0.00390625,"hidden-states-absmax/_layer_30_moe::router":0.9997426867485046,"hidden-states-absmax/_layer_12_moe::router":0.999682605266571,"update-rms/gated_delta::A_log":0.0006531372029275758,"hidden-states-absmax/_layer_22_pre_mlp_layernorm":2.796875,"hidden-states-l1/_layer_20_gated_delta::value":0.03041656119126811,"update-mean/_layer_39_gated_delta::out_norm":-0.0001220703125,"hidden-states-rms/_layer_07_residual_mlp":0.6853401954724626,"grad-absmax/layer_47/attention::linear_qkv":0.0003462103195488453,"grad-absmax/layer_11/moe::expert_fc2":1.5047383385535795e-05,"hidden-states-l1/_layer_17_gated_delta::core_attention":0.0005979467663328965,"hidden-states-rms/_layer_33_gated_delta::g":6.395044648927605,"update-l1/_layer_46_gated_delta::A_log":0.0004436657764017582,"param-mean/_layer_28_gated_delta::out_norm":0.795928955078125,"hidden-states-absmax/_layer_35_gated_delta::beta":12.4375,"hidden-states-l1/_layer_42_moe::router":0.01011896790964888,"hidden-states-mean/_layer_51_gated_delta::out_proj":0.23067445822649696,"hidden-states-absmax/_layer_42_gated_delta::out_norm":105.5,"hidden-states-mean/gated_delta::query":0.024245130308076155,"hidden-states-l1/_layer_32_gated_delta::in_proj":1.5040834689748759,"update-rms/_layer_20_gated_delta::dt_bias":0.000746065983882966,"moe/layer_37/act_rms_min":0.07601352781057358,"hidden-states-rms/_layer_14_input_layernorm":0.29955377467824773,"param-absmax/_layer_12_gated_delta::dt_bias":0.656865119934082,"update-absmax/_layer_07_gated_delta::conv1d":0.003021240234375,"grad-absmax/layer_39/moe::shared_expert_fc2":2.1250023564789444e-05,"hidden-states-rms/_layer_35_input_layernorm":0.5687253971278999,"param-mean/_layer_02_mlp::linear_fc1":9.87321891443571e-06,"moe/layer_04/act_rms_min":0.004575607366859913,"hidden-states-rms/_layer_41_pre_mlp_layernorm":0.22274215268914646,"hidden-states-absmax/_layer_16_moe::shared_experts":5.625,"hidden-states-absmax/_layer_02_gated_delta::gate":12.0625,"hidden-states-l1/_layer_10_input_layernorm":0.16515209259415778,"param-rms/_layer_41_gated_delta::out_proj":0.04344751221631193,"moe/layer_19/act_rms_max":0.10512453317642212,"update-mean/_layer_17_gated_delta::out_norm":-0.0002899169921875,"grad-rms/layer_24/moe::shared_experts":1.6850296473171108e-06,"hidden-states-absmax/_layer_39_mlp":27.625,"moe/layer_30/act_rms_min_to_median":0.4304623554174798,"hidden-states-absmax/_layer_43_gated_delta::query":23.625,"hidden-states-rms/_layer_33_gated_delta::key":0.2120674337640613,"hidden-states-mean/_layer_13_input_layernorm":0.0029201154818523506,"hidden-states-l1/_layer_37_gated_delta::out_proj":0.22790083025703792,"param-absmax/_layer_38_gated_delta::out_norm":1.78125,"hidden-states-absmax/_layer_12_gated_delta::key":6.21875,"hidden-states-absmax/_layer_15_pre_mlp_layernorm":3.15625,"param-absmax/_layer_26_gated_delta::conv1d":0.640625,"hidden-states-rms/_layer_10_gated_delta::value":0.04831005198291168,"grad-rms/layer_05/gated_delta::A_log":1.529721922895383e-06,"hidden-states-absmax/_layer_45_gated_delta::key":12.0625,"param-rms/_layer_33_gated_delta::out_proj":0.04460008314418554,"hidden-states-l1/_layer_53_pre_mlp_layernorm":0.1400547045534497,"hidden-states-mean/_layer_18_moe::shared_experts":1.876208667379999e-05,"hidden-states-rms/_layer_16_input_layernorm":0.29172298514184575,"update-l1/_layer_56_gated_delta::out_norm":0.0001220703125,"update-l1/_layer_03_gated_delta::out_norm":0.00049591064453125,"hidden-states-mean/_layer_03_gated_delta::g":-3.1766180010578635,"update-rms/_layer_18_gated_delta::dt_bias":0.0005390807779467647,"hidden-states-mean/_layer_27_gated_delta::value":0.001836187890760647,"hidden-states-l1/_layer_22_residual_mlp":0.8724693503939331,"hidden-states-l1/_layer_28_gated_delta::key":0.027685487217961217,"hidden-states-rms/_layer_14_gated_delta::query":0.18183913774158375,"grad-absmax/layer_34/gated_delta::out_proj":9.716414933791384e-05,"moe/layer_44/act_rms_min_to_median":0.3067962366552642,"hidden-states-l1/_layer_33_pre_mlp_layernorm":0.13818493642627436,"param-rms/_layer_04_gated_delta::dt_bias":0.4151519980767413,"update-rms/_layer_49_attention::linear_qkv":0.00018809400457512712,"hidden-states-l1/_layer_22_residual_attn":0.8139776961776848,"grad-rms/layer_44/gated_delta::out_norm":9.57433525089485e-06,"hidden-states-absmax/_layer_21_attention::o_proj":13.375,"grad-absmax/layer_50/moe::shared_expert_fc1":9.520541789242998e-05,"param-rms/_layer_23_gated_delta::out_proj":0.042333195523894786,"hidden-states-l1/_layer_48_gated_delta::beta":2.2314613141371384,"hidden-states-rms/_layer_18_gated_delta::value":0.04071473937967219,"hidden-states-rms/_layer_29_gated_delta::query":0.4432877902420206,"moe/layer_48/act_rms_min_to_median":0.3356540991580147,"update-l1/_layer_52_gated_delta::out_norm":0.00067138671875,"param-l1/_layer_11_gated_delta::out_proj":0.03486074507236481,"update-rms/_layer_30_gated_delta::out_proj":0.00021265727755019118,"moe/layer_12/act_rms_median":0.04182838648557663,"grad-rms/layer_28/moe::shared_expert_fc1":2.809222928773726e-07,"update-rms/_layer_19_gated_delta::out_norm":0.0019683246455807005,"hidden-states-mean/_layer_18_gated_delta::beta":-0.7862793548440165,"hidden-states-mean/_layer_15_gated_delta::g":-5.180456779057596,"update-mean/_layer_24_attention::linear_qkv":6.504956928665706e-08,"update-absmax/_layer_15_gated_delta::out_proj":0.00390625,"hidden-states-mean/_layer_52_gated_delta::out_proj":-0.007372437923841229,"hidden-states-absmax/_layer_49_moe::routed_experts":33.5,"update-mean/_layer_23_gated_delta::conv1d":-6.7523360485211015e-06,"grad-rms/layer_12/moe::shared_expert_fc2":4.3649968192452167e-07,"hidden-states-rms/gated_delta::key":0.1656305859708789,"hidden-states-l1/_layer_32_residual_mlp":1.8508349479240116,"hidden-states-l1/_layer_17_gated_delta::gate":0.7680270991357683,"hidden-states-mean/_layer_50_input_layernorm":0.008103709702635967,"hidden-states-mean/final_layernorm":0.0009165148877668247,"hidden-states-absmax/_layer_43_gated_delta::value":11.3125,"hidden-states-absmax/_layer_46_residual_attn":502,"hidden-states-rms/_layer_03_gated_delta::in_proj":1.1673703513814013,"hidden-states-l1/_layer_25_gated_delta::query":0.1218686100087893,"hidden-states-mean/_layer_03_gated_delta::query":0.007723018050702578,"grad-rms/layer_27/moe::shared_expert_fc2":1.498032196163329e-07,"update-mean/_layer_40_gated_delta::A_log":-1.561385579407215e-05,"param-rms/_layer_44_gated_delta::A_log":1.237108704326172,"update-rms/_layer_27_gated_delta::A_log":0.0006378009824121547,"hidden-states-rms/_layer_21_mlp":0.2144665742685999,"moe/layer_28/act_rms_min":0.04017956554889679,"hidden-states-rms/_layer_30_mlp":0.7716845748925858,"hidden-states-rms/_layer_01_input_layernorm":0.5074158815696133,"grad-rms/layer_08/moe::router":3.475774346816837e-07,"hidden-states-mean/_layer_08_residual_attn":0.001334329178365073,"hidden-states-absmax/_layer_25_gated_delta::key":7.0625,"hidden-states-absmax/_layer_45_gated_delta::beta":12.25,"hidden-states-absmax/_layer_38_gated_delta::beta":8.6875,"param-absmax/_layer_12_gated_delta::out_proj":1.390625,"param-l1/_layer_34_gated_delta::dt_bias":0.2928086519241333,"grad-absmax/layer_41/moe::expert_fc1":1.095849484045175e-06,"grad-rms/layer_15/gated_delta::conv1d":1.7102640198440552e-05,"param-l1/_layer_52_gated_delta::out_norm":0.891357421875,"param-l1/_layer_16_gated_delta::conv1d":0.03718699514865875,"hidden-states-absmax/_layer_20_gated_delta::gate":9.625,"update-absmax/_layer_52_gated_delta::out_norm":0.0078125,"param-rms/_layer_29_gated_delta::out_norm":0.41540041283682017,"hidden-states-mean/_layer_02_gated_delta::out_norm":0.0003946267713440831,"hidden-states-l1/_layer_53_gated_delta::out_norm":0.23095762502641975,"update-rms/_layer_39_gated_delta::dt_bias":0.0007480543161476848,"update-rms/_layer_08_gated_delta::dt_bias":0.0009652126990874737,"moe/layer_00/act_rms_min":0,"hidden-states-absmax/_layer_35_residual_attn":502,"hidden-states-absmax/_layer_21_moe::router":0.9976983070373535,"grad-absmax/layer_55/gated_delta::conv1d":0.009088972583413124,"param-absmax/_layer_47_attention::o_proj":0.60546875,"update-l1/_layer_01_gated_delta::out_proj":0.00012856944522354752,"param-mean/_layer_12_gated_delta::conv1d":-0.0011277544545009732,"grad-rms/layer_36/gated_delta::dt_bias":0.0001068366333887357,"hidden-states-absmax/_layer_52_mlp":111.5,"hidden-states-l1/_layer_41_gated_delta::value":0.047994988172648925,"param-l1/_layer_45_gated_delta::out_proj":0.03332306444644928,"update-rms/_layer_20_gated_delta::conv1d":0.0007947653920874632,"grad-absmax/layer_34/gated_delta::dt_bias":1.370910194964381e-05,"hidden-states-mean/_layer_55_residual_mlp":0.09237113951100583,"update-mean/_layer_45_gated_delta::out_proj":-1.3299356282914232e-08,"param-absmax/_layer_11_gated_delta::A_log":1.9216164350509644,"hidden-states-mean/_layer_34_gated_delta::gate":-0.5498234092978727,"hidden-states-absmax/_layer_15_residual_mlp":494,"hidden-states-absmax/_layer_55_moe::shared_experts":65,"grad-absmax/layer_54/moe::router":9.61181431193836e-05,"hidden-states-rms/_layer_23_gated_delta::gate":1.1588966052531757,"update-mean/_layer_12_gated_delta::out_proj":-1.8647229538260035e-08,"grad-rms/layer_38/moe::shared_expert_fc1":1.6423709021884676e-07,"hidden-states-rms/_layer_34_residual_attn":6.08328778003541,"param-l1/_layer_01_gated_delta::A_log":1.3519622087478638,"hidden-states-l1/_layer_09_gated_delta::out_proj":0.06935183040521507,"hidden-states-rms/_layer_48_residual_attn":9.855565715905023,"param-mean/_layer_40_gated_delta::A_log":1.029429316520691,"hidden-states-mean/_layer_51_gated_delta::query":0.09899453508788765,"grad-absmax/layer_46/gated_delta::conv1d":0.006510454695671797,"grad-absmax/layer_30/moe::router":0.00010567396384431049,"param-l1/_layer_31_gated_delta::dt_bias":0.2585441768169403,"hidden-states-mean/_layer_04_residual_mlp":0.0015964050119182482,"hidden-states-l1/_layer_56_gated_delta::beta":2.883912430275968,"param-mean/_layer_08_gated_delta::conv1d":0.0007053119479678571,"hidden-states-rms/_layer_09_mlp":0.09134023326236423,"hidden-states-l1/_layer_29_gated_delta::beta":2.4068162926216474,"param-absmax/_layer_12_gated_delta::conv1d":0.4453125,"hidden-states-l1/_layer_52_gated_delta::in_proj":2.011199010326151,"update-l1/_layer_56_gated_delta::conv1d":0.0004839028406422585,"hidden-states-absmax/_layer_33_moe::shared_experts":14.875,"update-absmax/_layer_32_gated_delta::A_log":0.0017960667610168457,"param-l1/_layer_25_gated_delta::out_proj":0.03467706963419914,"hidden-states-l1/_layer_04_gated_delta::beta":1.3377970859762534,"grad-absmax/layer_23/gated_delta::out_norm":4.115650517633185e-05,"grad-rms/layer_16/moe::shared_expert_fc1":3.2805334592458325e-07,"hidden-states-absmax/_layer_14_mlp":11.125,"hidden-states-absmax/_layer_54_gated_delta::key":6.21875,"hidden-states-absmax/_layer_43_moe::router":0.999221682548523,"hidden-states-mean/_layer_19_gated_delta::out_norm":-0.00011768004280798647,"grad-absmax/layer_26/gated_delta::out_proj":2.7709025744115934e-05,"update-mean/_layer_03_gated_delta::out_norm":-5.340576171875e-05,"hidden-states-l1/_layer_54_gated_delta::gate":1.3146924155583877,"param-mean/_layer_20_gated_delta::A_log":1.0600939989089966,"hidden-states-absmax/_layer_31_gated_delta::out_proj":16.5,"hidden-states-rms/_layer_08_gated_delta::beta":1.2404348395195328,"grad-absmax/layer_44/moe::shared_expert_fc1":2.935758857347537e-05,"grad-rms/layer_34/gated_delta::dt_bias":2.737088123620229e-06,"hidden-states-mean/_layer_38_gated_delta::query":0.02014704847598654,"update-l1/_layer_33_gated_delta::A_log":0.000643620325718075,"grad-rms/layer_18/moe::router":3.751623264046405e-07,"update-rms/_layer_39_gated_delta::out_norm":0.0015440808887540916,"update-rms/attention::o_proj":0.0002204614805295431,"hidden-states-mean/_layer_26_gated_delta::core_attention":-1.2422574062158335e-05,"hidden-states-absmax/_layer_05_gated_delta::out_norm":13.375,"hidden-states-rms/_layer_39_gated_delta::out_norm":0.24461133908840996,"param-mean/_layer_42_gated_delta::dt_bias":0.24730361998081207,"hidden-states-rms/_layer_20_pre_mlp_layernorm":0.12250906456816402,"hidden-states-absmax/_layer_23_gated_delta::query":8.0625,"moe/layer_15/act_rms_min":0.015394746325910091,"update-absmax/_layer_07_gated_delta::out_proj":0.00390625,"hidden-states-mean/_layer_08_pre_mlp_layernorm":-7.961120939835421e-05,"hidden-states-mean/_layer_31_gated_delta::gate":-0.38860111100686917,"hidden-states-absmax/_layer_37_moe::router":0.9998992681503296,"grad-absmax/layer_31/gated_delta::conv1d":0.00014776072930544615,"hidden-states-mean/_layer_30_gated_delta::beta":-0.6839140872852907,"hidden-states-mean/_layer_35_gated_delta::beta":-2.2009825946071855,"param-absmax/_layer_26_gated_delta::out_proj":1.15625,"grad-absmax/layer_15/moe::shared_expert_fc2":1.5619652913301252e-06,"update-l1/_layer_02_gated_delta::out_norm":0.000457763671875,"hidden-states-mean/_layer_36_gated_delta::beta":-0.5730021682980261,"grad-absmax/layer_53/gated_delta::A_log":1.9956669348175637e-05,"hidden-states-mean/_layer_10_gated_delta::key":0.011965980317113437,"grad-rms/layer_14/moe::expert_fc2":2.5523752514128645e-08,"update-mean/_layer_02_mlp::linear_fc2":-1.1026715185380453e-07,"param-mean/_layer_10_gated_delta::conv1d":0.0001615793298697099,"update-absmax/_layer_51_gated_delta::conv1d":0.00390625,"grad-rms/layer_40/gated_delta::out_proj":3.379132829328568e-07,"update-l1/_layer_19_gated_delta::conv1d":0.0006218881462700665,"param-rms/_layer_14_gated_delta::out_norm":0.42151695528579475,"hidden-states-absmax/_layer_12_gated_delta::out_norm":20.5,"hidden-states-mean/_layer_26_residual_attn":-0.015886208073640832,"param-rms/_layer_25_gated_delta::A_log":1.0281000870582078,"hidden-states-l1/_layer_51_gated_delta::query":0.1961598332118789,"hidden-states-rms/_layer_49_pre_mlp_layernorm":0.28451484625531287,"hidden-states-mean/_layer_01_gated_delta::beta":0.4378741905539263,"grad-absmax/layer_40/gated_delta::conv1d":0.0005023593548685312,"param-absmax/_layer_16_gated_delta::A_log":1.657900094985962,"param-mean/_layer_41_gated_delta::conv1d":-0.0009190855198539793,"hidden-states-absmax/_layer_12_residual_attn":484,"hidden-states-rms/_layer_32_residual_attn":5.703620288519339,"param-l1/_layer_08_gated_delta::out_norm":0.3342742919921875,"grad-absmax/layer_22/attention::o_proj":3.214597745682113e-05,"grad-absmax/layer_43/gated_delta::A_log":1.3000040780752897e-05,"grad-absmax/layer_04/gated_delta::out_norm":7.752397505100816e-05,"update-rms/_layer_30_gated_delta::conv1d":0.0008178835983975129,"hidden-states-mean/_layer_08_gated_delta::out_proj":-0.00017170377259895475,"hidden-states-rms/_layer_19_pre_mlp_layernorm":0.12998911502419302,"hidden-states-l1/_layer_24_moe::shared_experts":0.1825033643845754,"param-mean/_layer_46_gated_delta::dt_bias":0.5893617272377014,"hidden-states-mean/_layer_11_gated_delta::value":-0.0006521515510747195,"hidden-states-rms/_layer_41_gated_delta::out_proj":0.49964984907616045,"grad-absmax/layer_35/moe::shared_expert_fc2":7.406529675790807e-06,"update-rms/_layer_54_gated_delta::out_norm":0.001953125,"hidden-states-rms/_layer_28_gated_delta::out_norm":0.2938444957203573,"hidden-states-mean/_layer_53_gated_delta::beta":0.2750172048827654,"hidden-states-rms/_layer_40_gated_delta::core_attention":0.005614791544603573,"hidden-states-mean/_layer_15_gated_delta::beta":-1.1493053614265158,"grad-rms/layer_26/gated_delta::out_proj":4.4398600967430373e-07,"update-absmax/_layer_35_gated_delta::out_proj":0.001953125,"hidden-states-l1/_layer_32_gated_delta::out_norm":0.06896849798780656,"hidden-states-rms/_layer_55_gated_delta::beta":3.6521443413653096,"update-absmax/_layer_14_gated_delta::out_proj":0.00390625,"hidden-states-l1/_layer_51_moe::routed_experts":0.11175412199873232,"param-absmax/_layer_52_gated_delta::dt_bias":1.0194472074508667,"update-rms/attention::linear_qkv":0.00018696317160844415,"update-rms/lm_head":0.0002740292023238406,"grad-rms/layer_05/moe::router":4.371849449612913e-07,"hidden-states-rms/_layer_38_gated_delta::query":0.15486081518446032,"hidden-states-mean/_layer_12_gated_delta::out_norm":-0.00033574200279542837,"param-mean/_layer_04_gated_delta::out_proj":7.382623152807355e-06,"hidden-states-mean/_layer_04_gated_delta::out_proj":-2.9710401480953573e-05,"hidden-states-mean/_layer_01_gated_delta::out_proj":0.0005517575849080187,"grad-rms/layer_19/gated_delta::dt_bias":8.03566360060206e-06,"hidden-states-absmax/_layer_01_input_layernorm":13.25,"hidden-states-mean/_layer_54_moe::routed_experts":-8.432717606214442e-05,"hidden-states-l1/_layer_47_attention::core_attention":0.47601666760643335,"grad-rms/layer_27/gated_delta::conv1d":1.2715733706878315e-05,"hidden-states-rms/_layer_56_gated_delta::gate":2.9020494935057077,"update-absmax/_layer_49_attention::o_proj":0.001953125,"hidden-states-l1/_layer_26_moe::shared_experts":0.09218076970347996,"hidden-states-l1/_layer_16_gated_delta::value":0.028306515536488955,"param-mean/_layer_43_gated_delta::A_log":0.9358468651771545,"hidden-states-l1/_layer_20_gated_delta::key":0.07315694667779764,"hidden-states-rms/_layer_36_pre_mlp_layernorm":0.2013135642246371,"hidden-states-mean/_layer_46_moe::shared_experts":-0.00587516906859506,"hidden-states-mean/gated_delta::out_norm":-0.002070437112529275,"param-mean/_layer_35_gated_delta::dt_bias":0.2421136051416397,"hidden-states-rms/_layer_33_residual_mlp":5.998575080938348,"hidden-states-rms/_layer_28_gated_delta::out_proj":0.6184838726939272,"hidden-states-l1/_layer_27_gated_delta::core_attention":0.0008430805596656275,"update-mean/_layer_10_gated_delta::conv1d":-1.3522134395316243e-05,"moe/layer_44/act_rms_max":0.5558564066886902,"grad-rms/layer_44/gated_delta::out_proj":4.2781425222279596e-07,"hidden-states-rms/_layer_26_gated_delta::gate":1.1873749320705178,"hidden-states-l1/_layer_06_residual_mlp":0.2431576839873414,"hidden-states-l1/_layer_45_pre_mlp_layernorm":0.1929095766436549,"hidden-states-l1/_layer_37_gated_delta::beta":1.0788596962523098,"grad-absmax/layer_47/moe::expert_fc1":6.124384526629001e-06,"hidden-states-mean/_layer_31_moe::shared_experts":-0.0009091801395407485,"hidden-states-rms/_layer_53_residual_attn":25.428127667779062,"hidden-states-absmax/_layer_34_gated_delta::out_norm":186,"param-absmax/_layer_06_gated_delta::dt_bias":0.5851321816444397,"grad-rms/layer_24/moe::expert_fc2":2.1593409050743654e-08,"hidden-states-rms/_layer_06_residual_attn":0.6285854393401992,"hidden-states-mean/_layer_19_mlp":0.00041088594830911737,"update-rms/_layer_43_gated_delta::out_norm":0.001760523083722651,"param-rms/_layer_35_gated_delta::out_norm":0.8984220752221249,"hidden-states-absmax/_layer_11_gated_delta::in_proj":32.25,"param-absmax/_layer_01_mlp::linear_fc1":0.640625,"hidden-states-absmax/_layer_41_moe::shared_experts":49.25,"moe/layer_39/act_rms_min_to_median":0.5658353583983728,"update-absmax/_layer_41_gated_delta::conv1d":0.00390625,"grad-rms/layer_21/moe::shared_experts":1.3822020918797273e-06,"hidden-states-l1/_layer_48_gated_delta::key":0.086645449596101,"hidden-states-rms/_layer_41_gated_delta::out_norm":0.26069636549346614,"param-l1/_layer_22_attention::linear_qkv":0.027673454955220222,"hidden-states-absmax/_layer_52_gated_delta::key":5.125,"param-l1/_layer_01_gated_delta::conv1d":0.0356539748609066,"hidden-states-rms/_layer_06_gated_delta::in_proj":1.542170151320628,"hidden-states-rms/moe::routed_experts":0.17896307045134086,"grad-rms/layer_04/gated_delta::A_log":3.731992658447413e-06,"update-absmax/_layer_40_gated_delta::A_log":0.0017682313919067383,"hidden-states-mean/_layer_03_gated_delta::in_proj":-0.05442233657105402,"hidden-states-mean/_layer_18_gated_delta::query":0.013178746855705602,"grad-absmax/layer_18/moe::router":4.9475562263978645e-05,"param-mean/_layer_40_gated_delta::dt_bias":0.29485777020454407,"hidden-states-mean/_layer_23_mlp":-0.0002000796905348081,"hidden-states-mean/_layer_51_gated_delta::in_proj":-0.06711358832028126,"param-absmax/_layer_24_attention::linear_qkv":0.74609375,"hidden-states-absmax/_layer_19_moe::routed_experts":21.625,"moe/layer_29/act_rms_min":0.040668364614248276,"grad-rms/layer_39/gated_delta::conv1d":1.569976471474892e-05,"hidden-states-rms/_layer_31_gated_delta::in_proj":1.6330566848651524,"update-absmax/_layer_40_gated_delta::dt_bias":0.0017669200897216797,"param-rms/_layer_30_gated_delta::out_proj":0.04518809165656287,"hidden-states-mean/_layer_47_residual_attn":-0.09899659144837754,"hidden-states-rms/_layer_25_gated_delta::query":0.24736804105266,"grad-absmax/layer_37/gated_delta::out_norm":4.085551336174831e-05,"param-l1/_layer_20_gated_delta::out_norm":1.0211181640625,"param-rms/_layer_06_gated_delta::dt_bias":0.40469928194076105,"update-absmax/_layer_17_gated_delta::conv1d":0.00390625,"hidden-states-absmax/_layer_27_gated_delta::out_proj":12.8125,"hidden-states-l1/_layer_31_mlp":0.216085996161222,"hidden-states-rms/_layer_53_gated_delta::value":0.14336693946113865,"hidden-states-absmax/_layer_34_mlp":29.75,"grad-absmax/layer_11/moe::shared_experts":3.256632226111833e-06,"hidden-states-mean/_layer_25_mlp":-0.0024927284746144385,"hidden-states-rms/_layer_42_moe::shared_experts":0.2596446318918594,"hidden-states-mean/_layer_32_residual_attn":-0.03672744026511195,"hidden-states-mean/_layer_32_gated_delta::in_proj":-0.16407054515608283,"param-rms/_layer_56_gated_delta::dt_bias":0.8993329105774451,"param-absmax/_layer_33_gated_delta::dt_bias":0.5392111539840698,"hidden-states-l1/_layer_24_attention::k_layernorm":3.7835402230668334,"param-absmax/_layer_03_gated_delta::dt_bias":0.8089135885238647,"param-absmax/_layer_55_gated_delta::dt_bias":1.327911138534546,"grad-absmax/layer_25/gated_delta::out_proj":2.6772495402838103e-05,"grad-rms/layer_08/moe::shared_expert_fc1":2.34275352140138e-07,"moe/layer_53/act_rms_min":0.060905516147613525,"update-l1/_layer_26_gated_delta::A_log":0.00053459033370018,"hidden-states-mean/_layer_48_residual_attn":-0.10339476655659277,"hidden-states-l1/_layer_11_pre_mlp_layernorm":0.07480441925578347,"param-absmax/gated_delta::dt_bias":1.4029160737991333,"grad-rms/layer_25/moe::shared_expert_fc1":2.983592007897246e-07,"update-absmax/mlp::linear_fc1":0.00390625,"hidden-states-l1/_layer_11_gated_delta::out_proj":0.07882181749603957,"hidden-states-rms/_layer_35_gated_delta::beta":2.973264952012673,"hidden-states-mean/_layer_28_gated_delta::value":0.0038715254008353235,"update-absmax/_layer_33_gated_delta::A_log":0.0015660524368286133,"hidden-states-mean/_layer_01_gated_delta::value":0.09744143157499213,"hidden-states-rms/_layer_09_gated_delta::out_norm":0.04398189714716785,"grad-rms/layer_23/moe::shared_experts":2.0562383424855543e-06,"moe/layer_27/act_rms_median":0.09259947389364243,"grad-rms/layer_51/gated_delta::A_log":6.463955686321262e-06,"hidden-states-l1/_layer_35_gated_delta::beta":2.401378679439864,"hidden-states-absmax/_layer_18_gated_delta::in_proj":13.8125,"hidden-states-mean/_layer_07_pre_mlp_layernorm":-5.149768799120007e-06,"hidden-states-l1/_layer_41_gated_delta::in_proj":1.0390991436145116,"hidden-states-absmax/_layer_03_mlp":5.15625,"grad-rms/layer_51/moe::expert_fc2":1.765553222203891e-08,"grad-rms/layer_45/moe::expert_fc1":1.9784639954850863e-08,"param-mean/_layer_44_gated_delta::conv1d":-0.00034870277158915997,"update-absmax/gated_delta::dt_bias":0.00258713960647583,"param-rms/_layer_12_gated_delta::out_proj":0.043306810342566844,"hidden-states-rms/_layer_05_gated_delta::out_proj":0.07743589811909513,"hidden-states-l1/_layer_29_gated_delta::core_attention":0.0026017987011282087,"hidden-states-mean/_layer_42_gated_delta::out_proj":0.0006467013925106994,"hidden-states-absmax/_layer_18_gated_delta::beta":9.0625,"update-absmax/_layer_54_gated_delta::dt_bias":0.0011535286903381348,"hidden-states-rms/_layer_03_gated_delta::out_proj":0.0718094284336926,"update-l1/_layer_09_gated_delta::A_log":0.0006165127269923687,"update-rms/_layer_48_gated_delta::out_proj":0.00019322519527372855,"param-mean/_layer_05_gated_delta::out_norm":0.2126312255859375,"grad-absmax/layer_48/gated_delta::conv1d":0.02040976472198963,"update-rms/_layer_42_gated_delta::out_proj":0.00020947578167089027,"hidden-states-mean/attention::o_proj":0.0008854371154084343,"hidden-states-rms/moe::router":0.07216878508256834,"update-absmax/_layer_51_gated_delta::A_log":0.0018368959426879883,"hidden-states-l1/_layer_21_residual_attn":0.753728837219338,"param-mean/_layer_10_gated_delta::dt_bias":-0.015434607863426208,"hidden-states-rms/_layer_14_gated_delta::beta":3.206577027843208,"update-rms/_layer_23_gated_delta::out_proj":0.0001963925560042886,"grad-rms/layer_32/gated_delta::dt_bias":7.074298858741478e-07,"param-absmax/_layer_23_gated_delta::A_log":2.006493091583252,"hidden-states-l1/_layer_17_gated_delta::value":0.03246415048410302,"hidden-states-mean/_layer_47_input_layernorm":0.01241563036032085,"hidden-states-rms/_layer_01_gated_delta::g":12.655603313004036,"update-l1/_layer_30_gated_delta::A_log":0.0004395092837512493,"hidden-states-l1/_layer_10_gated_delta::in_proj":0.7006818701362016,"hidden-states-mean/_layer_26_gated_delta::out_proj":-0.003094664212425257,"hidden-states-absmax/_layer_54_gated_delta::out_norm":144,"hidden-states-mean/_layer_02_mlp":2.2977997115751383e-05,"hidden-states-l1/_layer_26_residual_mlp":1.2787406776834342,"param-absmax/_layer_21_attention::o_proj":0.83984375,"hidden-states-absmax/_layer_14_gated_delta::out_norm":21.875,"hidden-states-l1/_layer_42_moe::routed_experts":0.10881994961751154,"hidden-states-mean/_layer_10_residual_mlp":0.0005333799107676638,"hidden-states-l1/_layer_29_gated_delta::query":0.128343791168188,"update-rms/_layer_53_gated_delta::dt_bias":0.0006739246020322653,"hidden-states-rms/attention::k_layernorm":4.256666882451615,"hidden-states-rms/_layer_47_residual_attn":9.550754659210288,"update-rms/_layer_52_gated_delta::conv1d":0.000781665956172514,"update-absmax/_layer_53_gated_delta::A_log":0.0018963813781738281,"grad-rms/attention::linear_qkv":1.9067808573444991e-06,"param-rms/mlp::linear_fc1":0.04541706858087636,"param-l1/_layer_23_gated_delta::out_norm":0.728668212890625,"hidden-states-mean/_layer_33_gated_delta::core_attention":0.00013522397941785806,"update-absmax/_layer_09_gated_delta::out_proj":0.0078125,"hidden-states-rms/_layer_18_gated_delta::gate":1.0979065093463265,"grad-rms/layer_09/moe::shared_expert_fc2":1.605160788417026e-07,"param-l1/_layer_53_gated_delta::A_log":1.3193913698196411,"hidden-states-mean/_layer_51_input_layernorm":0.003364514960588808,"grad-absmax/layer_16/gated_delta::out_proj":2.305549060110934e-05,"param-mean/_layer_05_gated_delta::conv1d":-0.00045778811909258366,"hidden-states-mean/_layer_53_pre_mlp_layernorm":-0.0011567442194216513,"param-mean/_layer_05_gated_delta::A_log":1.3300670385360718,"update-mean/_layer_41_gated_delta::out_proj":3.659162572944297e-08,"moe/layer_05/act_rms_min":0.0058695063926279545,"hidden-states-l1/_layer_47_moe::routed_experts":0.16544166398611618,"grad-rms/layer_56/moe::expert_fc1":1.313015354521673e-08,"update-absmax/_layer_48_gated_delta::conv1d":0.0030670166015625,"update-l1/_layer_01_gated_delta::A_log":0.0005677641020156443,"grad-rms/layer_02/mlp::linear_fc2":1.6289473459226253e-07,"hidden-states-absmax/_layer_24_residual_mlp":500,"param-absmax/_layer_54_gated_delta::dt_bias":0.9880474805831909,"grad-rms/layer_27/moe::router":6.118410998030382e-07,"moe/layer_09/act_rms_min":0.010363290086388588,"grad-absmax/layer_14/moe::router":6.597708124900237e-05,"grad-rms/layer_40/gated_delta::out_norm":1.0121785813044108e-05,"hidden-states-l1/_layer_37_gated_delta::query":0.058668627878263935,"grad-rms/layer_37/moe::expert_fc2":2.0341923541472508e-08,"grad-absmax/layer_53/moe::expert_fc1":2.942052742582746e-05,"grad-absmax/layer_30/moe::shared_expert_fc1":1.9056871678913012e-05,"update-rms/_layer_18_gated_delta::A_log":0.0005368440232762017,"hidden-states-absmax/_layer_29_input_layernorm":27.875,"hidden-states-rms/_layer_43_gated_delta::out_proj":0.6145709505210569,"param-rms/_layer_32_gated_delta::out_proj":0.04537919486161671,"hidden-states-mean/_layer_56_gated_delta::in_proj":-0.6068955981309301,"hidden-states-rms/_layer_08_gated_delta::value":0.03628510738637518,"hidden-states-l1/_layer_34_moe::shared_experts":0.11137585179517887,"update-absmax/_layer_17_gated_delta::dt_bias":0.0015908628702163696,"hidden-states-rms/_layer_36_moe::router":0.07216878508256834,"hidden-states-mean/_layer_01_mlp":0.00016562152241780877,"hidden-states-mean/_layer_18_gated_delta::in_proj":0.015032463547738491,"hidden-states-rms/_layer_35_gated_delta::out_norm":0.19823359257044815,"hidden-states-l1/attention::q_layernorm":1.1544904552431308,"update-l1/_layer_44_gated_delta::A_log":0.0005229373928159475,"hidden-states-absmax/_layer_30_gated_delta::core_attention":0.1142578125,"update-rms/_layer_26_gated_delta::out_norm":0.0025371838001497227,"update-mean/_layer_53_gated_delta::dt_bias":-0.00024701980873942375,"hidden-states-rms/_layer_40_gated_delta::in_proj":1.9598205988915116,"param-l1/_layer_28_gated_delta::conv1d":0.028291506692767143,"hidden-states-mean/_layer_08_mlp":-0.00017597521133704767,"update-mean/_layer_32_gated_delta::out_norm":-0.0001068115234375,"update-mean/_layer_01_gated_delta::out_norm":0.0001678466796875,"hidden-states-absmax/_layer_28_pre_mlp_layernorm":2.796875,"grad-rms/layer_44/gated_delta::A_log":1.0605479235765025e-05,"hidden-states-rms/_layer_15_moe::routed_experts":0.04993786543256862,"moe/layer_54/act_rms_min":0.0427895113825798,"update-rms/_layer_37_gated_delta::dt_bias":0.000627983471701242,"update-mean/_layer_38_gated_delta::A_log":-0.00022047932725399733,"update-rms/_layer_20_gated_delta::A_log":0.0007301530351341959,"grad-absmax/layer_54/moe::expert_fc1":1.9968749711551936e-06,"hidden-states-rms/_layer_37_gated_delta::core_attention":0.001984648957318103,"param-rms/_layer_11_gated_delta::A_log":1.2449555176138212,"grad-rms/layer_29/gated_delta::dt_bias":1.4409761943629838e-06,"hidden-states-l1/_layer_25_gated_delta::in_proj":0.9104733794858372,"hidden-states-l1/_layer_10_moe::shared_experts":0.04262707518065872,"hidden-states-rms/_layer_18_mlp":0.15147717982427553,"update-mean/_layer_11_gated_delta::dt_bias":-9.043496538652107e-05,"param-rms/_layer_19_gated_delta::A_log":1.2649038638649803,"hidden-states-absmax/_layer_06_input_layernorm":16.375,"update-mean/_layer_43_gated_delta::A_log":-4.266912583261728e-05,"hidden-states-absmax/_layer_50_input_layernorm":43.75,"hidden-states-mean/_layer_47_moe::routed_experts":-0.0005729494252954596,"param-mean/_layer_35_gated_delta::out_norm":0.842559814453125,"param-mean/_layer_31_gated_delta::out_proj":2.4159877284546383e-05,"param-rms/_layer_53_gated_delta::conv1d":0.06466259297176737,"hidden-states-rms/_layer_44_gated_delta::beta":2.0306705564970673,"moe/layer_18/act_rms_min":0.023493606597185135,"param-rms/_layer_12_gated_delta::dt_bias":0.40937185213465754,"hidden-states-absmax/_layer_13_moe::router":0.998357355594635,"hidden-states-rms/_layer_17_gated_delta::core_attention":0.0013578780968386479,"hidden-states-mean/_layer_42_gated_delta::core_attention":5.108192153366837e-05,"hidden-states-l1/_layer_05_gated_delta::in_proj":0.7325276252395843,"update-mean/_layer_44_gated_delta::A_log":0.000132259214296937,"update-l1/_layer_38_gated_delta::dt_bias":0.00044663273729383945,"grad-rms/layer_04/moe::shared_experts":2.6371138645623065e-07,"hidden-states-rms/_layer_33_mlp":0.37151884735744517,"update-l1/_layer_42_gated_delta::out_norm":0.00042724609375,"hidden-states-l1/_layer_14_gated_delta::out_proj":0.10597748082607757,"hidden-states-mean/_layer_30_residual_attn":-0.030237603811932065,"hidden-states-absmax/_layer_06_gated_delta::out_proj":29.625,"hidden-states-rms/_layer_14_gated_delta::g":8.264792880387015,"param-absmax/_layer_21_attention::linear_qkv":0.8125,"hidden-states-absmax/_layer_43_residual_attn":502,"param-rms/_layer_45_gated_delta::out_proj":0.043012194724217,"update-l1/_layer_55_gated_delta::out_norm":0.0006103515625,"update-rms/_layer_20_gated_delta::out_proj":0.0001946370851809452,"hidden-states-rms/_layer_29_moe::shared_experts":0.9704039170748627,"update-absmax/_layer_01_gated_delta::out_proj":0.00390625,"update-l1/_layer_45_gated_delta::conv1d":0.000621038256213069,"update-mean/_layer_54_gated_delta::out_proj":-8.853753286075516e-08,"update-mean/_layer_23_gated_delta::dt_bias":-0.00010854919673874974,"grad-rms/layer_17/moe::shared_experts":9.761286382546481e-07,"hidden-states-l1/_layer_15_residual_attn":0.4789935027353171,"update-l1/_layer_01_gated_delta::conv1d":0.0006090255919843912,"hidden-states-absmax/_layer_03_gated_delta::g":17.75,"update-absmax/_layer_46_gated_delta::out_proj":0.00390625,"update-mean/_layer_31_gated_delta::out_proj":1.5400279096411396e-07,"hidden-states-l1/_layer_20_residual_mlp":0.7288387094543826,"hidden-states-absmax/_layer_37_moe::shared_experts":11.9375,"hidden-states-absmax/_layer_07_residual_attn":56,"hidden-states-l1/_layer_35_gated_delta::in_proj":1.1557768038315765,"hidden-states-absmax/_layer_56_gated_delta::gate":42,"grad-absmax/layer_42/moe::expert_fc1":9.07219964574324e-06,"grad-absmax/layer_52/gated_delta::dt_bias":7.011953857727349e-05,"param-rms/_layer_09_gated_delta::conv1d":0.06578509395970841,"param-l1/_layer_25_gated_delta::dt_bias":0.18906238675117493,"hidden-states-rms/_layer_46_pre_mlp_layernorm":0.26030732204876733,"param-rms/_layer_26_gated_delta::out_norm":0.9873394908109868,"update-absmax/_layer_21_attention::o_proj":0.00390625,"hidden-states-mean/_layer_01_pre_mlp_layernorm":0.00013556017235545633,"grad-absmax/layer_03/moe::router":0.00012508635700214654,"moe/layer_35/act_rms_min":0.0705418661236763,"hidden-states-mean/_layer_52_residual_attn":0.13291010354877753,"hidden-states-l1/_layer_04_residual_attn":0.19708769882770763,"grad-absmax/layer_05/moe::shared_expert_fc1":1.552526737214066e-05,"hidden-states-mean/_layer_43_gated_delta::query":0.06445343070667156,"grad-absmax/layer_17/moe::router":4.1195435187546536e-05,"grad-absmax/layer_43/moe::shared_expert_fc2":6.278733508224832e-06,"grad-rms/layer_35/gated_delta::conv1d":1.2381458735791008e-05,"update-rms/_layer_38_gated_delta::out_proj":0.00020887876071269137,"grad-absmax/layer_27/gated_delta::dt_bias":7.3051219260378275e-06,"hidden-states-mean/_layer_45_gated_delta::out_norm":0.0011147539063249715,"hidden-states-absmax/_layer_34_gated_delta::in_proj":32,"update-l1/_layer_08_gated_delta::A_log":0.0007223319262266159,"update-mean/_layer_17_gated_delta::conv1d":5.262274044071091e-06,"param-mean/_layer_32_gated_delta::dt_bias":0.22803524136543274,"hidden-states-mean/_layer_55_gated_delta::value":-0.002133236986635564,"hidden-states-absmax/_layer_44_gated_delta::beta":11.375,"param-mean/_layer_34_gated_delta::conv1d":-0.0011348356492817402,"hidden-states-rms/_layer_51_residual_mlp":24.225816945595756,"grad-absmax/layer_56/gated_delta::A_log":3.5119508538628e-05,"param-mean/_layer_02_gated_delta::out_norm":0.14124298095703125,"hidden-states-absmax/_layer_29_moe::routed_experts":18.375,"hidden-states-l1/_layer_51_moe::router":0.010305283987368581,"hidden-states-rms/_layer_52_moe::router":0.07216878508256834,"param-l1/_layer_19_gated_delta::dt_bias":0.13379524648189545,"grad-rms/layer_45/moe::shared_expert_fc1":3.4217886376757407e-07,"hidden-states-rms/_layer_10_moe::router":0.07216878508256834,"grad-rms/layer_54/moe::shared_experts":1.5065396351064968e-06,"param-rms/_layer_04_gated_delta::A_log":1.1697146020348823,"hidden-states-l1/_layer_19_pre_mlp_layernorm":0.09834392796086407,"update-rms/_layer_29_gated_delta::out_norm":0.0013756625573223442,"hidden-states-rms/_layer_56_pre_mlp_layernorm":0.14633692867743,"param-rms/_layer_38_gated_delta::conv1d":0.049494026905614254,"hidden-states-l1/_layer_45_gated_delta::key":0.1554153895873296,"param-mean/_layer_54_gated_delta::dt_bias":0.6362190842628479,"hidden-states-l1/_layer_27_gated_delta::key":0.08274786415670683,"hidden-states-mean/_layer_32_mlp":-0.001915635303781665,"moe/layer_05/act_rms_max":0.057150326669216156,"hidden-states-absmax/_layer_47_attention::k_layernorm":42.5,"update-l1/_layer_34_gated_delta::conv1d":0.0006111122784204781,"param-mean/_layer_51_gated_delta::out_proj":-4.8450110625708476e-05,"param-l1/_layer_06_gated_delta::A_log":1.307487964630127,"hidden-states-l1/_layer_03_gated_delta::value":0.038551130271092686,"hidden-states-l1/_layer_34_residual_mlp":2.0378186747811644,"hidden-states-l1/_layer_44_gated_delta::out_norm":0.16542765980096577,"hidden-states-absmax/_layer_46_gated_delta::g":19.875,"param-absmax/_layer_14_gated_delta::out_norm":0.72265625,"hidden-states-absmax/_layer_05_moe::routed_experts":9.6875,"hidden-states-rms/_layer_27_gated_delta::in_proj":1.2606361868861635,"update-l1/_layer_16_gated_delta::dt_bias":0.0005225341883487999,"param-absmax/_layer_27_gated_delta::out_proj":0.78515625,"hidden-states-mean/_layer_06_input_layernorm":0.0032135179837516526,"hidden-states-absmax/_layer_31_residual_attn":502,"param-mean/_layer_15_gated_delta::out_norm":0.504150390625,"update-l1/_layer_01_gated_delta::dt_bias":0.0006065985653549433,"hidden-states-mean/_layer_55_pre_mlp_layernorm":-0.0008425182735941736,"hidden-states-absmax/_layer_36_residual_attn":502,"param-absmax/embedding":3.15625,"hidden-states-l1/_layer_35_gated_delta::out_proj":0.25887635456041747,"hidden-states-rms/_layer_04_gated_delta::beta":1.9687772438975004,"grad-absmax/layer_39/moe::router":2.4433127691736445e-05,"hidden-states-absmax/_layer_05_moe::router":0.9996805787086487,"update-mean/_layer_07_gated_delta::dt_bias":2.4281558580696583e-05,"hidden-states-l1/_layer_51_pre_mlp_layernorm":0.16336627761915656,"hidden-states-rms/_layer_17_gated_delta::key":0.1821462511662384,"moe/layer_23/act_rms_max":0.21589067578315735,"update-absmax/_layer_51_gated_delta::out_proj":0.001953125,"hidden-states-l1/_layer_33_moe::router":0.010240630943080734,"param-mean/_layer_38_gated_delta::conv1d":-0.00041168282041326165,"hidden-states-l1/_layer_41_gated_delta::out_proj":0.30677718600954845,"grad-absmax/layer_46/gated_delta::out_proj":0.00011511532648000866,"hidden-states-l1/_layer_23_gated_delta::key":0.09871224000599861,"hidden-states-mean/_layer_13_attention::o_proj":0.00065183199097821,"hidden-states-l1/_layer_21_mlp":0.15147207992936135,"update-l1/_layer_03_gated_delta::A_log":0.0006105790380388498,"hidden-states-mean/_layer_23_moe::router":0.010258910945480815,"hidden-states-l1/_layer_08_gated_delta::g":2.97256052166945,"grad-rms/layer_30/gated_delta::out_norm":8.660192700850636e-06,"param-l1/_layer_44_gated_delta::out_proj":0.032768622040748596,"hidden-states-absmax/_layer_36_gated_delta::g":20.5,"param-rms/_layer_44_gated_delta::out_proj":0.04261142315635185,"grad-absmax/layer_27/gated_delta::out_norm":2.7974729164270684e-05,"hidden-states-rms/gated_delta::g":6.4924142088093,"grad-rms/layer_34/moe::shared_expert_fc2":1.4189632531566483e-07,"update-absmax/_layer_36_gated_delta::dt_bias":0.0012390613555908203,"hidden-states-rms/_layer_28_moe::router":0.07216878508256834,"hidden-states-mean/_layer_33_gated_delta::out_norm":-0.001285942424648097,"hidden-states-absmax/_layer_16_moe::router":0.9999693632125854,"grad-absmax/layer_10/gated_delta::dt_bias":1.1509491741890088e-05,"hidden-states-l1/_layer_29_residual_attn":1.4338815729476897,"hidden-states-l1/_layer_13_moe::shared_experts":0.06196688854299647,"param-rms/_layer_53_gated_delta::dt_bias":0.7229280527704136,"hidden-states-mean/_layer_33_residual_mlp":-0.045618122293725284,"hidden-states-rms/_layer_35_gated_delta::out_proj":0.42214906454156503,"param-l1/_layer_09_gated_delta::out_proj":0.03507346287369728,"update-absmax/_layer_55_gated_delta::dt_bias":0.001508176326751709,"hidden-states-rms/_layer_18_moe::routed_experts":0.0594727233104668,"param-absmax/_layer_33_gated_delta::A_log":1.8057224750518799,"update-absmax/_layer_41_gated_delta::out_norm":0.00390625,"hidden-states-absmax/_layer_45_pre_mlp_layernorm":6.625,"hidden-states-absmax/_layer_49_moe::router":0.9987143874168396,"grad-absmax/layer_41/moe::shared_experts":1.0316944099031389e-05,"grad-rms/layer_54/moe::shared_expert_fc1":1.5486109146589606e-07,"hidden-states-absmax/_layer_29_gated_delta::in_proj":29.5,"hidden-states-mean/_layer_29_gated_delta::key":-0.002490993886112438,"hidden-states-rms/_layer_22_residual_mlp":1.8187629524863904,"hidden-states-mean/_layer_36_gated_delta::key":0.009435800097680103,"hidden-states-absmax/_layer_31_gated_delta::core_attention":0.185546875,"grad-absmax/layer_54/moe::shared_expert_fc1":5.482793767441763e-06,"param-absmax/_layer_33_gated_delta::out_norm":0.69140625,"param-rms/_layer_16_gated_delta::dt_bias":0.1500488397937774,"hidden-states-rms/_layer_43_gated_delta::out_norm":0.3128361034856197,"param-rms/_layer_31_gated_delta::out_proj":0.04648959146609682,"param-absmax/_layer_02_gated_delta::out_norm":0.39453125,"hidden-states-absmax/_layer_54_gated_delta::gate":26.375,"grad-rms/layer_51/gated_delta::out_norm":3.0513550616698582e-05,"moe/layer_13/act_rms_median":0.042802900075912476,"hidden-states-mean/_layer_25_gated_delta::beta":-1.4336838047127491,"moe/layer_02/act_rms_min":0.0028808836359530687,"grad-rms/layer_55/moe::expert_fc1":1.3835524635012643e-08,"hidden-states-absmax/_layer_11_residual_attn":478,"hidden-states-absmax/_layer_28_residual_attn":502,"grad-rms/layer_40/gated_delta::A_log":2.012934390878631e-06,"hidden-states-rms/_layer_09_gated_delta::out_proj":0.09462164026211593,"hidden-states-rms/_layer_28_gated_delta::query":0.07412447527940774,"grad-absmax/layer_12/moe::expert_fc1":1.1030353562091477e-05,"hidden-states-l1/_layer_44_moe::router":0.010180323450616749,"hidden-states-mean/_layer_34_gated_delta::out_norm":-0.0021804536498133717,"grad-rms/layer_27/moe::expert_fc2":2.1725081392428763e-08,"update-mean/_layer_41_gated_delta::A_log":0.0002194219414377585,"update-mean/_layer_41_gated_delta::dt_bias":0.00022793027164880186,"hidden-states-mean/_layer_02_gated_delta::gate":-0.04289448400074053,"grad-rms/layer_06/moe::shared_experts":5.340586354145709e-07,"hidden-states-l1/_layer_01_gated_delta::core_attention":0.013443524689200483,"grad-absmax/layer_06/moe::shared_expert_fc2":2.111591584252892e-06,"grad-absmax/layer_07/gated_delta::conv1d":0.00025407574139535427,"hidden-states-l1/_layer_03_gated_delta::core_attention":0.001215094379644916,"hidden-states-mean/_layer_14_gated_delta::out_norm":-0.00019147653901856112,"hidden-states-mean/_layer_40_pre_mlp_layernorm":0.0004177181897386241,"update-l1/_layer_56_gated_delta::dt_bias":0.0006101047620177269,"hidden-states-mean/_layer_09_residual_attn":0.0009866276535096948,"grad-rms/mlp::linear_fc1":2.692709409964489e-07,"hidden-states-l1/_layer_29_input_layernorm":0.29457602436256697,"grad-absmax/layer_17/gated_delta::conv1d":0.0003844253078568727,"hidden-states-mean/_layer_55_moe::routed_experts":-0.00014951804328965253,"param-absmax/_layer_43_gated_delta::out_norm":1.2890625,"param-l1/_layer_31_gated_delta::out_norm":0.911224365234375,"update-absmax/_layer_06_gated_delta::A_log":0.0013117790222167969,"hidden-states-mean/_layer_23_moe::shared_experts":0.0001071018032580901,"grad-absmax/layer_32/gated_delta::out_norm":5.420856905402616e-05,"hidden-states-absmax/_layer_31_pre_mlp_layernorm":3.203125,"param-l1/_layer_17_gated_delta::out_proj":0.033411890268325806,"hidden-states-absmax/_layer_12_gated_delta::g":22.75,"param-rms/_layer_32_gated_delta::A_log":1.2137990304804431,"grad-rms/layer_03/gated_delta::A_log":5.975405121105633e-06,"grad-rms/layer_33/moe::shared_expert_fc2":1.4833217676638306e-07,"hidden-states-absmax/final_layernorm":56.75,"param-rms/attention::linear_qkv":0.03902431968002482,"grad-rms/layer_26/gated_delta::dt_bias":2.015143934714307e-06,"hidden-states-l1/_layer_26_gated_delta::value":0.02835052053338534,"hidden-states-rms/_layer_24_attention::o_proj":0.6215123030661576,"hidden-states-l1/_layer_34_gated_delta::out_norm":0.07198378556773491,"hidden-states-l1/_layer_12_moe::shared_experts":0.08083635042147046,"hidden-states-rms/_layer_13_mlp":0.13076794045026122,"hidden-states-absmax/_layer_01_gated_delta::g":70.5,"moe/layer_31/act_rms_min_to_median":0.4017585783960467,"hidden-states-l1/_layer_04_gated_delta::g":3.675730666605314,"grad-rms/layer_24/moe::router":7.791632643265573e-07,"update-absmax/_layer_29_gated_delta::conv1d":0.0029296875,"update-absmax/_layer_20_gated_delta::dt_bias":0.00258713960647583,"hidden-states-rms/_layer_05_gated_delta::out_norm":0.03520508240585885,"grad-absmax/layer_09/moe::expert_fc2":1.006755724120012e-06,"hidden-states-mean/_layer_52_gated_delta::in_proj":-0.06095811452588819,"hidden-states-l1/_layer_11_moe::router":0.010233603802697468,"hidden-states-absmax/_layer_08_moe::router":0.9981181621551514,"hidden-states-rms/_layer_16_moe::shared_experts":0.07804655535426457,"hidden-states-mean/_layer_27_moe::shared_experts":-0.0005478266464749451,"hidden-states-rms/_layer_45_moe::shared_experts":0.47730319234092167,"hidden-states-mean/_layer_45_pre_mlp_layernorm":0.0003134935187428914,"hidden-states-rms/_layer_03_gated_delta::out_norm":0.031096158905867387,"hidden-states-rms/_layer_08_gated_delta::key":0.09792018142068469,"update-l1/_layer_38_gated_delta::out_norm":0.0003662109375,"hidden-states-mean/_layer_20_gated_delta::query":0.022358395961927888,"hidden-states-l1/_layer_44_gated_delta::query":0.09110863102777374,"hidden-states-rms/_layer_14_moe::shared_experts":0.07538215984589743,"hidden-states-rms/_layer_02_residual_attn":0.5041563615923126,"grad-rms/layer_26/moe::shared_expert_fc1":2.6954888373008205e-07,"grad-rms/layer_13/moe::router":4.4681503269960226e-07,"hidden-states-mean/_layer_09_residual_mlp":0.0009663053193940603,"hidden-states-absmax/_layer_04_gated_delta::out_proj":4.65625,"hidden-states-absmax/_layer_29_gated_delta::out_proj":18,"hidden-states-absmax/_layer_36_moe::shared_experts":20.125,"hidden-states-mean/_layer_30_input_layernorm":0.0007639748256549031,"param-rms/_layer_20_gated_delta::out_proj":0.041697833051594906,"hidden-states-absmax/_layer_09_gated_delta::in_proj":33,"hidden-states-l1/_layer_14_gated_delta::query":0.07274095091115843,"hidden-states-absmax/_layer_51_gated_delta::out_norm":832,"grad-rms/moe::expert_fc2":2.0346735569256666e-08,"hidden-states-l1/_layer_31_gated_delta::gate":1.0416145374315426,"hidden-states-l1/_layer_44_gated_delta::out_proj":0.5639321880259738,"update-absmax/_layer_29_gated_delta::out_norm":0.00390625,"hidden-states-mean/_layer_26_input_layernorm":0.0004421096472908136,"hidden-states-l1/_layer_56_residual_attn":6.757489056287891,"hidden-states-mean/_layer_47_pre_mlp_layernorm":0.0002929075329957197,"hidden-states-l1/_layer_24_pre_mlp_layernorm":0.1245740055530078,"param-rms/_layer_39_gated_delta::A_log":1.3039047268629462,"hidden-states-l1/_layer_44_gated_delta::key":0.10958143139856236,"hidden-states-mean/_layer_38_gated_delta::gate":-0.4500436127237812,"hidden-states-l1/_layer_45_gated_delta::beta":2.018342497151196,"hidden-states-l1/_layer_35_gated_delta::out_norm":0.07387640462832507,"hidden-states-l1/_layer_21_moe::shared_experts":0.10490937291495477,"hidden-states-absmax/_layer_42_mlp":32,"hidden-states-rms/_layer_33_moe::shared_experts":0.19415345759989805,"hidden-states-absmax/_layer_17_gated_delta::out_norm":15.5,"hidden-states-l1/_layer_46_gated_delta::g":4.641150074142226,"hidden-states-l1/_layer_07_gated_delta::g":4.175498506486238,"hidden-states-mean/_layer_49_pre_mlp_layernorm":0.0004919597999852657,"hidden-states-absmax/_layer_07_gated_delta::in_proj":21.625,"grad-absmax/layer_39/moe::expert_fc1":1.1828497008536942e-06,"update-absmax/_layer_31_gated_delta::out_proj":0.001953125,"param-absmax/_layer_38_gated_delta::conv1d":0.4375,"hidden-states-rms/_layer_04_moe::routed_experts":0.0191966268282358,"hidden-states-rms/_layer_40_moe::routed_experts":0.1704980202778271,"hidden-states-rms/_layer_22_moe::router":0.07216878508256834,"update-absmax/_layer_11_gated_delta::conv1d":0.0030517578125,"update-l1/_layer_04_gated_delta::dt_bias":0.0003085094504058361,"hidden-states-absmax/_layer_55_moe::routed_experts":131,"hidden-states-rms/_layer_53_gated_delta::gate":1.9974345200821964,"hidden-states-l1/_layer_09_gated_delta::gate":0.8819323637404876,"hidden-states-absmax/_layer_03_pre_mlp_layernorm":3.265625,"hidden-states-mean/_layer_13_attention::k_layernorm":0.012048256157130737,"hidden-states-mean/_layer_09_gated_delta::gate":-0.42418663790066696,"param-rms/_layer_43_gated_delta::dt_bias":0.37799931163321493,"hidden-states-l1/_layer_33_moe::routed_experts":0.07543662876178493,"update-absmax/_layer_32_gated_delta::conv1d":0.00390625,"update-l1/_layer_45_gated_delta::A_log":0.00046593183651566505,"update-l1/_layer_17_gated_delta::out_proj":0.00014217324496712536,"moe/layer_32/act_rms_max":0.2949921786785126,"param-rms/_layer_51_gated_delta::out_proj":0.044159327032730504,"hidden-states-l1/_layer_49_attention::o_proj":0.6731159297366058,"hidden-states-absmax/_layer_25_pre_mlp_layernorm":3.828125,"grad-absmax/layer_45/gated_delta::A_log":0.00010858022142201662,"hidden-states-l1/_layer_13_attention::core_attention":0.11478030559781857,"grad-absmax/layer_55/moe::expert_fc1":2.912654963438399e-06,"hidden-states-rms/_layer_14_gated_delta::key":0.16533702131653658,"grad-rms/layer_06/moe::expert_fc1":1.5926133230136446e-08,"update-absmax/_layer_12_gated_delta::dt_bias":0.0010838359594345093,"grad-absmax/layer_45/moe::expert_fc2":3.228012928957469e-06,"hidden-states-absmax/_layer_53_moe::shared_experts":278,"hidden-states-l1/_layer_14_gated_delta::out_norm":0.03325118520870032,"hidden-states-l1/_layer_38_gated_delta::g":2.143467920239916,"param-l1/_layer_44_gated_delta::dt_bias":0.5241995453834534,"hidden-states-mean/_layer_16_residual_attn":-0.00043425830601254774,"hidden-states-l1/_layer_35_gated_delta::g":7.246474176224968,"hidden-states-mean/_layer_10_gated_delta::query":0.013089244882970438,"grad-rms/layer_38/moe::expert_fc2":2.0815572448578136e-08,"hidden-states-mean/_layer_07_gated_delta::in_proj":-0.19331283977299818,"hidden-states-absmax/_layer_49_attention::k_layernorm":32.25,"hidden-states-l1/_layer_10_gated_delta::core_attention":0.0006137600434583477,"hidden-states-l1/_layer_12_gated_delta::g":5.617298319017687,"hidden-states-rms/_layer_25_gated_delta::key":0.2582422643338118,"hidden-states-absmax/_layer_17_moe::routed_experts":4.6875,"hidden-states-mean/_layer_14_input_layernorm":0.0008286822527379993,"hidden-states-absmax/_layer_44_gated_delta::out_proj":67,"hidden-states-absmax/_layer_45_gated_delta::gate":42.25,"hidden-states-absmax/_layer_38_gated_delta::gate":15.4375,"update-absmax/_layer_45_gated_delta::A_log":0.0014748871326446533,"hidden-states-rms/_layer_13_input_layernorm":0.44293374909816136,"grad-rms/layer_26/gated_delta::conv1d":1.6649586704967347e-05,"hidden-states-l1/gated_delta::query":0.08631310067150527,"param-l1/mlp::linear_fc2":0.03379715420305729,"hidden-states-rms/_layer_12_gated_delta::g":6.483171631999344,"moe/layer_46/act_rms_median":0.26838919520378113,"update-mean/_layer_30_gated_delta::dt_bias":-0.00012477314157877117,"update-rms/_layer_14_gated_delta::out_proj":0.0002088914146508272,"hidden-states-mean/_layer_21_moe::routed_experts":-5.5456136885457856e-05,"param-l1/_layer_40_gated_delta::out_proj":0.03398614749312401,"hidden-states-l1/_layer_44_moe::shared_experts":0.2445085489164053,"grad-rms/layer_15/moe::router":4.778629359691319e-07,"hidden-states-absmax/_layer_02_residual_attn":18.5,"param-mean/_layer_41_gated_delta::out_norm":1.004638671875,"param-mean/_layer_55_gated_delta::out_proj":2.5943560103769414e-05,"hidden-states-l1/_layer_42_input_layernorm":0.2104929787508224,"param-absmax/_layer_27_gated_delta::out_norm":1.625,"hidden-states-rms/_layer_52_gated_delta::key":0.23832960746257598,"hidden-states-mean/embeddings":-0.0005608510725763319,"grad-rms/layer_07/moe::expert_fc2":1.8318166994336423e-08,"param-absmax/_layer_14_gated_delta::out_proj":1.265625,"hidden-states-absmax/_layer_38_gated_delta::in_proj":23.75,"grad-rms/layer_11/moe::shared_expert_fc2":1.9932088431406218e-07,"hidden-states-rms/_layer_38_input_layernorm":0.43659012602787306,"hidden-states-absmax/_layer_20_gated_delta::beta":11.1875,"update-absmax/_layer_26_gated_delta::conv1d":0.00390625,"hidden-states-l1/residual_attn":1.94619057111798,"hidden-states-rms/_layer_24_moe::routed_experts":0.11706051226999378,"hidden-states-rms/_layer_56_moe::shared_experts":3.5077337295115303,"hidden-states-l1/_layer_39_moe::router":0.010072172668126848,"grad-absmax/layer_31/moe::expert_fc1":3.4501877053116914e-06,"param-absmax/_layer_45_gated_delta::dt_bias":0.8733481168746948,"hidden-states-absmax/_layer_22_moe::router":0.9942854642868042,"hidden-states-absmax/_layer_43_pre_mlp_layernorm":4.875,"update-l1/_layer_31_gated_delta::conv1d":0.000673182716127485,"param-rms/_layer_10_gated_delta::out_norm":0.5348537453272715,"update-l1/_layer_15_gated_delta::A_log":0.0005418272339738905,"moe/layer_16/act_rms_min":0.015411785803735256,"param-mean/_layer_18_gated_delta::out_norm":0.8354949951171875,"hidden-states-mean/_layer_27_gated_delta::beta":-1.6163153190988682,"hidden-states-rms/_layer_49_moe::shared_experts":0.523449943052951,"update-l1/_layer_16_gated_delta::out_proj":0.0001440007472410798,"update-mean/_layer_25_gated_delta::out_proj":5.624789878311276e-08,"update-mean/_layer_17_gated_delta::A_log":0.00013774937542621046,"hidden-states-mean/_layer_06_gated_delta::beta":-0.915984680173955,"hidden-states-l1/_layer_43_residual_attn":2.78733477441487,"hidden-states-rms/_layer_15_gated_delta::key":0.1515319018444994,"grad-rms/layer_36/gated_delta::out_proj":7.536282890472083e-07,"param-mean/_layer_13_attention::o_proj":-1.0465083505550865e-05,"hidden-states-mean/_layer_53_residual_mlp":0.11755022317769803,"param-absmax/gated_delta::conv1d":1.0859375,"hidden-states-absmax/_layer_09_gated_delta::query":5.53125,"grad-rms/layer_02/mlp::linear_fc1":3.600840822099214e-07,"hidden-states-absmax/_layer_32_gated_delta::out_norm":46,"global_load_balancing_loss":0.9990425948743467,"hidden-states-l1/_layer_24_attention::q_layernorm":1.1424691846930008,"param-rms/_layer_36_gated_delta::dt_bias":0.567095527863497,"update-mean/_layer_21_attention::linear_qkv":5.554796067563927e-10,"hidden-states-rms/_layer_26_gated_delta::key":0.10905685955010322,"grad-absmax/layer_36/gated_delta::A_log":0.0005759023479185998,"hidden-states-l1/_layer_09_input_layernorm":0.18183305046952372,"hidden-states-rms/_layer_34_input_layernorm":0.6287767165861425,"param-rms/gated_delta::conv1d":0.06772825959512023,"hidden-states-l1/_layer_35_gated_delta::value":0.047072449722582066,"update-absmax/_layer_05_gated_delta::A_log":0.0010764598846435547,"hidden-states-l1/_layer_54_gated_delta::value":0.05762851552707142,"hidden-states-absmax/_layer_06_residual_mlp":55,"hidden-states-rms/_layer_30_gated_delta::out_norm":0.18292619266042742,"param-absmax/_layer_54_gated_delta::A_log":2.3273754119873047,"grad-absmax/layer_34/moe::shared_expert_fc1":8.967012945504393e-06,"hidden-states-rms/_layer_14_residual_mlp":1.2056475589467937,"grad-rms/layer_09/gated_delta::conv1d":1.6991284960196194e-05,"param-rms/_layer_52_gated_delta::conv1d":0.08306893454497402,"update-absmax/_layer_01_mlp::linear_fc2":0.00390625,"hidden-states-mean/_layer_40_gated_delta::value":0.0018751355726433483,"param-l1/_layer_30_gated_delta::A_log":1.0956392288208008,"update-rms/_layer_19_gated_delta::conv1d":0.0007952737527896541,"hidden-states-absmax/_layer_56_gated_delta::g":20,"hidden-states-absmax/_layer_16_moe::routed_experts":29.5,"hidden-states-rms/_layer_43_residual_attn":8.045863773071156,"param-rms/_layer_26_gated_delta::conv1d":0.07271091419824981,"grad-rms/layer_40/gated_delta::dt_bias":1.967535047677753e-06,"hidden-states-mean/_layer_54_mlp":0.0006432807607016874,"hidden-states-l1/_layer_27_gated_delta::value":0.021206627333954255,"hidden-states-l1/_layer_23_gated_delta::value":0.034289144341843644,"hidden-states-absmax/_layer_47_attention::o_proj":34,"grad-rms/layer_49/attention::o_proj":3.224005635186255e-07,"hidden-states-mean/_layer_19_residual_mlp":-0.0012481693735026567,"moe/layer_17/act_rms_max":0.09960012882947922,"grad-absmax/layer_51/moe::shared_expert_fc2":2.7370782845537178e-05,"param-l1/_layer_19_gated_delta::out_proj":0.032480549067258835,"moe/layer_42/act_rms_min_to_median":0.41155931190054107,"param-absmax/_layer_02_gated_delta::out_proj":0.546875,"param-absmax/_layer_33_gated_delta::out_proj":0.796875,"grad-absmax/layer_04/gated_delta::conv1d":0.000284268957329914,"grad-absmax/layer_25/moe::shared_expert_fc2":4.273845661373343e-06,"hidden-states-absmax/_layer_18_moe::shared_experts":5.5625,"moe/layer_22/act_rms_median":0.08316995203495026,"update-mean/_layer_20_gated_delta::out_proj":2.9029305181893506e-08,"hidden-states-rms/_layer_54_moe::routed_experts":0.21982906437651437,"grad-absmax/layer_11/moe::router":3.417734114918858e-05,"update-rms/_layer_42_gated_delta::A_log":0.0005480053763506123,"param-mean/_layer_33_gated_delta::out_norm":0.4266357421875,"hidden-states-l1/_layer_32_pre_mlp_layernorm":0.1363398297238147,"hidden-states-absmax/_layer_11_gated_delta::g":32.25,"param-rms/_layer_36_gated_delta::out_proj":0.04135684718934121,"hidden-states-absmax/_layer_41_residual_mlp":502,"grad-rms/layer_08/gated_delta::A_log":1.8226430713994239e-06,"grad-rms/layer_35/moe::router":4.5031854180891697e-07,"hidden-states-l1/_layer_31_gated_delta::g":5.346612613989427,"param-rms/_layer_06_gated_delta::conv1d":0.058320868368751035,"hidden-states-rms/_layer_48_gated_delta::gate":1.5512120647794285,"update-mean/_layer_42_gated_delta::dt_bias":-9.948137449100614e-05,"hidden-states-absmax/_layer_46_gated_delta::in_proj":63.25,"grad-rms/layer_12/moe::expert_fc1":1.6503451255534157e-08,"update-absmax/_layer_30_gated_delta::out_norm":0.00390625,"param-absmax/_layer_17_gated_delta::conv1d":0.7734375,"hidden-states-rms/_layer_48_gated_delta::g":4.524338765458913,"moe/layer_34/act_rms_median":0.1305726170539856,"max_vio":0.20523230731487274,"hidden-states-l1/_layer_05_input_layernorm":0.18856843573670815,"param-rms/_layer_43_gated_delta::conv1d":0.07148360463024336,"param-mean/_layer_44_gated_delta::out_proj":6.221501826075837e-05,"hidden-states-rms/_layer_37_moe::router":0.07216878508256834,"hidden-states-rms/_layer_30_moe::router":0.07216878508256834,"hidden-states-mean/_layer_18_mlp":-0.00015379085758256706,"param-rms/_layer_50_attention::o_proj":0.05736640526496556,"hidden-states-rms/_layer_18_gated_delta::core_attention":0.0008009837908462935,"moe/layer_46/act_rms_min_to_median":0.31851835777295895,"hidden-states-rms/_layer_23_gated_delta::key":0.1861051003272442,"hidden-states-l1/_layer_26_gated_delta::g":4.899909603584957,"hidden-states-absmax/_layer_41_gated_delta::g":17.5,"param-l1/_layer_26_gated_delta::conv1d":0.03830060735344887,"param-mean/_layer_47_attention::linear_qkv":8.376187906833366e-05,"update-l1/_layer_23_gated_delta::dt_bias":0.0005235947319306433,"param-absmax/_layer_43_gated_delta::out_proj":0.86328125,"hidden-states-l1/_layer_51_input_layernorm":0.3838611404730249,"grad-rms/layer_18/gated_delta::out_norm":9.293668298736718e-06,"hidden-states-absmax/_layer_01_residual_mlp":12.5,"grad-rms/layer_04/moe::expert_fc2":1.3082570805205149e-08,"hidden-states-absmax/_layer_28_gated_delta::g":18.25,"update-mean/mlp::linear_fc1":-1.3761962236458203e-08,"update-l1/_layer_04_gated_delta::A_log":0.00031626049894839525,"hidden-states-l1/_layer_50_mlp":0.6484340010206701,"hidden-states-l1/_layer_48_gated_delta::in_proj":1.1395686716685987,"param-l1/_layer_27_gated_delta::dt_bias":0.1915915161371231,"update-rms/_layer_48_gated_delta::dt_bias":0.0006672531669239304,"hidden-states-mean/_layer_08_gated_delta::g":-2.9659758765668722,"hidden-states-l1/_layer_54_gated_delta::key":0.0847462516720595,"hidden-states-rms/_layer_46_gated_delta::out_norm":0.661708018125366,"hidden-states-mean/_layer_16_mlp":8.741731604274032e-05,"hidden-states-l1/_layer_14_moe::routed_experts":0.028783971381446605,"hidden-states-l1/_layer_11_gated_delta::out_norm":0.023962100098979484,"hidden-states-mean/_layer_03_residual_mlp":0.0014548954192796849,"hidden-states-mean/_layer_39_gated_delta::in_proj":-0.17939759990528248,"hidden-states-rms/_layer_30_residual_attn":4.721386789086723,"update-absmax/_layer_14_gated_delta::A_log":0.001504063606262207,"hidden-states-rms/_layer_07_gated_delta::key":0.086448631157151,"param-rms/_layer_34_gated_delta::out_norm":0.6301839419181913,"grad-absmax/layer_36/gated_delta::dt_bias":0.0005705449730157852,"hidden-states-absmax/_layer_55_residual_mlp":2816,"update-mean/_layer_19_gated_delta::out_norm":-0.000213623046875,"param-mean/gated_delta::out_norm":0.7485978262765067,"param-mean/_layer_04_gated_delta::dt_bias":0.36418259143829346,"hidden-states-absmax/_layer_44_moe::router":0.9977922439575195,"update-mean/_layer_43_gated_delta::conv1d":3.7284721656760667e-06,"param-l1/_layer_54_gated_delta::A_log":1.3945183753967285,"param-l1/_layer_26_gated_delta::A_log":0.9724692702293396,"hidden-states-absmax/_layer_19_gated_delta::out_proj":9.125,"hidden-states-rms/_layer_50_input_layernorm":0.7537974199566666,"hidden-states-rms/_layer_55_gated_delta::query":0.1769240109245098,"update-mean/_layer_39_gated_delta::out_proj":-7.91370595720764e-08,"hidden-states-rms/_layer_55_gated_delta::g":6.787427049574701,"hidden-states-rms/_layer_23_residual_mlp":2.0926254548836667,"hidden-states-absmax/_layer_42_gated_delta::core_attention":0.09423828125,"hidden-states-absmax/_layer_31_gated_delta::in_proj":25.375,"hidden-states-mean/_layer_30_gated_delta::core_attention":-9.847833362778342e-06,"update-l1/_layer_11_gated_delta::A_log":0.0005110702477395535,"grad-absmax/layer_19/moe::shared_expert_fc2":1.2549560779007152e-05,"hidden-states-rms/_layer_18_residual_attn":1.3565202681107504,"hidden-states-mean/_layer_12_gated_delta::beta":-1.0613473726925102,"update-absmax/_layer_27_gated_delta::A_log":0.001796424388885498,"grad-absmax/layer_34/moe::expert_fc1":2.053052639894304e-06,"hidden-states-absmax/_layer_00_embeddings":3.15625,"grad-absmax/layer_26/moe::expert_fc2":1.378041815769393e-06,"hidden-states-l1/_layer_02_residual_mlp":0.21552735156146902,"hidden-states-rms/_layer_56_residual_attn":28.49082781557941,"update-absmax/_layer_10_gated_delta::conv1d":0.00390625,"update-absmax/_layer_09_gated_delta::conv1d":0.0029296875,"hidden-states-rms/_layer_14_gated_delta::out_proj":0.17020294278079726,"grad-rms/layer_56/moe::shared_experts":3.562015804029124e-06,"hidden-states-rms/input_layernorm":0.5450752295955561,"hidden-states-rms/_layer_23_moe::routed_experts":0.09146389113383095,"hidden-states-mean/_layer_06_residual_mlp":0.0014200741953863734,"grad-absmax/layer_05/gated_delta::A_log":3.6760616239917e-06,"param-l1/_layer_33_gated_delta::A_log":0.9554508924484253,"hidden-states-mean/_layer_05_gated_delta::in_proj":-0.1430696104138796,"hidden-states-mean/_layer_14_gated_delta::value":9.073767786791806e-05,"param-mean/_layer_15_gated_delta::A_log":1.027811884880066,"update-rms/_layer_25_gated_delta::out_norm":0.0013593174713940482,"hidden-states-rms/_layer_11_gated_delta::in_proj":1.3328968950320017,"param-l1/_layer_35_gated_delta::out_proj":0.03433602675795555,"update-mean/_layer_17_gated_delta::out_proj":-1.978641819277982e-07,"hidden-states-rms/_layer_11_gated_delta::out_norm":0.05174968847167796,"hidden-states-mean/_layer_15_gated_delta::in_proj":-0.10396007057096417,"grad-rms/layer_54/moe::expert_fc2":1.321751421329751e-08,"_runtime":531833.312049525,"hidden-states-absmax/_layer_07_residual_mlp":58.5,"grad-rms/layer_20/moe::shared_expert_fc2":5.154462007932194e-07,"hidden-states-rms/_layer_34_gated_delta::out_proj":0.4674589776229411,"update-l1/_layer_28_gated_delta::conv1d":0.0005948045873083174,"hidden-states-rms/_layer_10_moe::shared_experts":0.33223791721000784,"update-l1/_layer_11_gated_delta::conv1d":0.0006486459169536829,"hidden-states-mean/_layer_27_residual_attn":-0.019826742390005873,"timing/save-checkpoint-sec":0,"hidden-states-mean/_layer_05_moe::router":0.010305154223069227,"update-l1/_layer_04_gated_delta::out_proj":0.00015318334044422954,"grad-rms/layer_12/moe::shared_experts":1.6226465850933052e-06,"update-l1/_layer_02_mlp::linear_fc1":0.00015836699458304793,"hidden-states-absmax/_layer_48_gated_delta::key":3,"hidden-states-absmax/_layer_03_gated_delta::in_proj":30.25,"param-mean/_layer_31_gated_delta::conv1d":0.000781699491199106,"hidden-states-l1/_layer_56_gated_delta::core_attention":0.004281634509609283,"hidden-states-mean/_layer_04_moe::routed_experts":1.2604162624540055e-05,"hidden-states-absmax/_layer_33_moe::routed_experts":22.125,"hidden-states-absmax/_layer_10_gated_delta::key":4.59375,"param-rms/_layer_46_gated_delta::out_norm":1.2147252555536159,"hidden-states-mean/_layer_37_moe::router":0.010114200158190752,"param-mean/_layer_46_gated_delta::out_norm":1.19207763671875,"hidden-states-rms/_layer_14_gated_delta::out_norm":0.0837316641990663,"param-mean/_layer_09_gated_delta::out_norm":0.36493682861328125,"grad-absmax/layer_40/moe::expert_fc2":9.280799986299826e-07,"hidden-states-mean/_layer_29_gated_delta::g":-8.561062879689082,"param-absmax/_layer_20_gated_delta::conv1d":1.0859375,"update-mean/gated_delta::dt_bias":-4.071327937857191e-05,"hidden-states-mean/_layer_55_residual_attn":0.08125994578122446,"hidden-states-rms/_layer_44_input_layernorm":0.6339958948533833,"hidden-states-absmax/_layer_38_residual_attn":502,"hidden-states-rms/_layer_09_pre_mlp_layernorm":0.08566305890565457,"update-absmax/lm_head":0.0078125,"update-l1/_layer_12_gated_delta::conv1d":0.0005935051012784243,"hidden-states-l1/_layer_42_gated_delta::core_attention":0.00035303203755812797,"grad-absmax/moe::shared_expert_fc2":7.731730147497728e-05,"hidden-states-mean/_layer_38_residual_mlp":-0.08275837692718231,"param-l1/_layer_48_gated_delta::dt_bias":0.6855649948120117,"hidden-states-rms/_layer_11_gated_delta::out_proj":0.10966947856615555,"hidden-states-mean/_layer_27_gated_delta::in_proj":-0.02586588102419639,"hidden-states-l1/_layer_05_gated_delta::value":0.03164161373872921,"hidden-states-mean/_layer_09_mlp":-0.00036778779777403743,"moe/layer_46/act_rms_max":0.6193366050720215,"hidden-states-mean/_layer_02_input_layernorm":0.002066520521896544,"hidden-states-rms/_layer_04_gated_delta::query":0.09083955343676975,"hidden-states-rms/_layer_17_mlp":0.14166333583775248,"hidden-states-rms/_layer_34_gated_delta::out_norm":0.2386606730650073,"grad-absmax/layer_09/moe::shared_expert_fc1":1.0980057595588733e-05,"hidden-states-rms/_layer_33_pre_mlp_layernorm":0.19480878930829515,"update-rms/_layer_29_gated_delta::A_log":0.0007813029220680171,"hidden-states-mean/_layer_22_moe::shared_experts":0.000797197550573653,"param-l1/_layer_21_attention::o_proj":0.027576787397265434,"hidden-states-absmax/_layer_25_gated_delta::beta":8.375,"hidden-states-absmax/_layer_15_gated_delta::beta":17.75,"hidden-states-absmax/_layer_43_gated_delta::out_norm":86,"hidden-states-absmax/_layer_04_gated_delta::gate":14.5,"hidden-states-absmax/_layer_18_moe::router":0.9903278350830078,"hidden-states-l1/_layer_38_gated_delta::gate":0.90070955761785,"grad-absmax/layer_01/gated_delta::A_log":0.00024655996821820736,"hidden-states-mean/_layer_20_residual_mlp":-0.001541826941402488,"hidden-states-mean/_layer_09_gated_delta::out_norm":-0.00017790946669169863,"hidden-states-l1/_layer_14_moe::router":0.01024958530061806,"update-absmax/_layer_19_gated_delta::A_log":0.0010993480682373047,"grad-absmax/layer_45/gated_delta::out_proj":0.00023695988056715578,"hidden-states-l1/_layer_32_gated_delta::out_proj":0.2551316028990272,"hidden-states-l1/_layer_07_gated_delta::beta":1.562770314546861,"hidden-states-mean/_layer_03_moe::router":0.010353417229202149,"update-rms/_layer_34_gated_delta::out_norm":0.0014850494458735887,"grad-rms/layer_50/moe::shared_experts":3.5070765359469735e-06,"hidden-states-mean/_layer_46_moe::router":0.010210930440573762,"hidden-states-mean/_layer_42_gated_delta::query":0.013316655256349475,"hidden-states-mean/_layer_52_gated_delta::beta":-0.24748879641160923,"grad-rms/layer_10/gated_delta::out_proj":4.1965058778521546e-07,"update-l1/_layer_20_gated_delta::A_log":0.0005439567030407488,"hidden-states-l1/_layer_46_mlp":0.5098535430256775,"param-l1/_layer_10_gated_delta::out_norm":0.50787353515625,"hidden-states-l1/_layer_14_gated_delta::g":7.773699733459917,"hidden-states-rms/_layer_31_pre_mlp_layernorm":0.18889857410920616,"grad-absmax/layer_39/moe::shared_experts":9.343877536593936e-06,"hidden-states-rms/final_layernorm":0.9025098186790947,"hidden-states-l1/_layer_34_gated_delta::out_proj":0.29947001331687134,"hidden-states-mean/_layer_10_gated_delta::out_norm":-0.00021996069411278594,"hidden-states-l1/_layer_32_gated_delta::beta":1.2497965395208048,"param-absmax/_layer_07_gated_delta::dt_bias":0.6703183650970459,"hidden-states-l1/_layer_31_input_layernorm":0.24305564082332143,"param-absmax/_layer_28_gated_delta::dt_bias":0.48469647765159607,"param-mean/_layer_02_mlp::linear_fc2":1.6168445654329844e-05,"update-absmax/gated_delta::conv1d":0.00390625,"update-rms/_layer_16_gated_delta::dt_bias":0.0006536647444919468,"update-absmax/_layer_06_gated_delta::out_norm":0.00390625,"hidden-states-rms/_layer_04_gated_delta::key":0.07333162371087965,"param-absmax/_layer_41_gated_delta::A_log":1.7984126806259155,"hidden-states-rms/_layer_48_gated_delta::value":0.08056893102500559,"moe/layer_41/act_rms_median":0.18366912007331848,"grad-absmax/layer_19/gated_delta::conv1d":0.0005711318226531148,"update-rms/_layer_03_gated_delta::out_proj":0.00021994435364873467,"hidden-states-absmax/_layer_27_moe::shared_experts":7.9375,"grad-absmax/layer_42/moe::router":6.133211718406528e-05,"grad-absmax/layer_55/moe::shared_experts":5.9771268752228934e-06,"hidden-states-mean/_layer_02_residual_attn":0.00018610245787642395,"hidden-states-l1/_layer_32_gated_delta::value":0.09088150750178188,"hidden-states-rms/_layer_46_gated_delta::out_proj":1.113026438618189,"grad-rms/layer_12/gated_delta::conv1d":9.426071450786415e-05,"update-absmax/_layer_50_attention::o_proj":0.0078125,"hidden-states-mean/_layer_01_residual_attn":-0.00053179177275798,"grad-absmax/layer_23/moe::shared_expert_fc1":6.288184522418305e-05,"hidden-states-l1/_layer_02_residual_attn":0.21372524961534137,"hidden-states-rms/_layer_32_gated_delta::in_proj":2.157525423658831,"hidden-states-mean/_layer_45_gated_delta::value":0.011102258619935348,"param-l1/_layer_03_gated_delta::dt_bias":0.4023759067058563,"hidden-states-l1/_layer_15_residual_mlp":0.49438196353697345,"update-mean/_layer_45_gated_delta::out_norm":-0.0003662109375,"hidden-states-rms/_layer_39_gated_delta::query":0.1851767421731441,"hidden-states-rms/_layer_41_gated_delta::value":0.09057028023809663,"grad-rms/layer_05/moe::expert_fc2":1.9557108542101903e-08,"grad-absmax/layer_41/gated_delta::out_proj":0.00012773927301168442,"param-absmax/_layer_28_gated_delta::A_log":2.022650957107544,"grad-absmax/layer_53/moe::shared_expert_fc1":0.0002041728002950549,"update-mean/_layer_12_gated_delta::out_norm":3.0517578125e-05,"param-absmax/_layer_05_gated_delta::A_log":2.0669398307800293,"hidden-states-mean/_layer_42_residual_attn":-0.09300596630644396,"moe/layer_17/act_rms_min_to_median":0.35511750086462607,"moe/layer_33/act_rms_min":0.061023108661174774,"update-l1/_layer_18_gated_delta::dt_bias":0.0004518108908087015,"param-absmax/_layer_35_gated_delta::conv1d":0.65234375,"update-absmax/_layer_27_gated_delta::out_norm":0.0078125,"hidden-states-absmax/_layer_18_residual_mlp":494,"hidden-states-l1/_layer_05_pre_mlp_layernorm":0.04408659939010136,"grad-absmax/layer_16/gated_delta::dt_bias":9.695771950646304e-06,"hidden-states-mean/_layer_34_gated_delta::key":0.016994262277395998,"update-absmax/_layer_55_gated_delta::out_norm":0.0078125,"update-l1/_layer_33_gated_delta::conv1d":0.0006495618727058172,"param-mean/_layer_54_gated_delta::conv1d":0.0008261938928626478,"update-mean/gated_delta::out_proj":-8.950392409083318e-09,"hidden-states-absmax/_layer_10_residual_mlp":71.5,"grad-absmax/layer_53/gated_delta::conv1d":0.011806503869593143,"moe/layer_54/act_rms_max":0.46412330865859985,"grad-rms/layer_09/gated_delta::A_log":1.949854762370379e-06,"hidden-states-absmax/_layer_02_gated_delta::beta":12.6875,"grad-rms/layer_11/moe::shared_experts":8.100817866519379e-07,"grad-rms/layer_41/moe::shared_expert_fc1":2.2064931439513582e-07,"hidden-states-mean/_layer_33_gated_delta::key":-0.001930240655383844,"moe/layer_10/act_rms_median":0.03774604573845863,"update-absmax/_layer_18_gated_delta::dt_bias":0.0011337734758853912,"hidden-states-mean/_layer_34_input_layernorm":0.0012906340790923333,"hidden-states-l1/_layer_48_gated_delta::core_attention":0.0012921676183095716,"hidden-states-rms/_layer_30_gated_delta::out_proj":0.4371065265555365,"grad-rms/layer_07/gated_delta::out_norm":2.8306062794139137e-05,"update-mean/_layer_03_gated_delta::out_proj":3.847872065421143e-08,"grad-rms/layer_13/moe::expert_fc1":1.9193190881264162e-08,"hidden-states-absmax/_layer_11_gated_delta::out_norm":18.25,"hidden-states-absmax/_layer_06_moe::router":0.9992082118988037,"hidden-states-rms/_layer_12_mlp":0.16178178247078814,"hidden-states-l1/_layer_19_mlp":0.13652388888882266,"param-mean/_layer_10_gated_delta::out_norm":0.50787353515625,"hidden-states-absmax/_layer_09_gated_delta::value":8,"param-rms/_layer_27_gated_delta::out_proj":0.04480868311329304,"hidden-states-l1/_layer_46_gated_delta::query":0.10077972421404412,"hidden-states-rms/_layer_20_gated_delta::beta":2.262208483488061,"hidden-states-absmax/_layer_03_gated_delta::out_proj":11.125,"hidden-states-l1/_layer_44_gated_delta::g":3.271038445358091,"hidden-states-l1/_layer_24_residual_mlp":1.1227742579906295,"grad-absmax/layer_11/moe::shared_expert_fc1":4.7189965698635206e-05,"hidden-states-l1/_layer_41_gated_delta::out_norm":0.08739941118948592,"hidden-states-absmax/_layer_49_moe::shared_experts":52.25,"param-absmax/_layer_51_gated_delta::A_log":1.9753960371017456,"hidden-states-rms/_layer_38_gated_delta::core_attention":0.0007675342198945813,"hidden-states-mean/_layer_39_moe::shared_experts":-0.0001578740672747424,"hidden-states-mean/_layer_28_gated_delta::gate":-0.04982354340833943,"hidden-states-l1/_layer_51_mlp":0.5747305243886143,"hidden-states-mean/_layer_23_pre_mlp_layernorm":-0.0006638957629712835,"param-mean/_layer_04_gated_delta::conv1d":-0.0010335148544982076,"hidden-states-mean/_layer_42_input_layernorm":0.0053880970828946925,"hidden-states-mean/_layer_04_gated_delta::gate":-0.26439038090591205,"hidden-states-l1/_layer_11_moe::shared_experts":0.04597770598265004,"grad-rms/layer_02/gated_delta::A_log":2.099076090518495e-06,"grad-rms/layer_28/gated_delta::dt_bias":5.056618813444029e-06,"hidden-states-l1/pre_mlp_layernorm":0.11777171226796483,"grad-rms/layer_41/gated_delta::out_norm":6.214830471781567e-06,"update-absmax/_layer_14_gated_delta::conv1d":0.00390625,"param-l1/_layer_41_gated_delta::out_norm":1.004638671875,"hidden-states-rms/_layer_33_gated_delta::gate":1.5656536492287245,"hidden-states-absmax/_layer_14_residual_attn":494,"hidden-states-absmax/_layer_35_gated_delta::gate":24.125,"param-l1/_layer_14_gated_delta::A_log":0.8586011528968811,"param-l1/_layer_51_gated_delta::out_proj":0.0339512825012207,"param-mean/_layer_34_gated_delta::A_log":0.9175292253494263,"update-l1/_layer_47_attention::o_proj":0.00018227487453259528,"hidden-states-rms/gated_delta::value":0.14267395082553044,"hidden-states-l1/_layer_55_gated_delta::core_attention":0.0028430399148809808,"param-mean/_layer_10_gated_delta::A_log":1.1486897468566895,"grad-rms/layer_37/moe::shared_expert_fc2":1.068469049341215e-07,"param-l1/_layer_31_gated_delta::conv1d":0.04334468021988869,"param-mean/_layer_33_gated_delta::A_log":0.8597379922866821,"update-absmax/_layer_20_gated_delta::A_log":0.0025861263275146484,"hidden-states-absmax/_layer_54_gated_delta::core_attention":0.5859375,"hidden-states-absmax/_layer_30_gated_delta::gate":10.875,"hidden-states-mean/_layer_14_gated_delta::beta":-2.4714918698658788,"hidden-states-absmax/_layer_07_gated_delta::gate":19.625,"update-rms/_layer_42_gated_delta::conv1d":0.0007779153974253728,"hidden-states-l1/_layer_11_input_layernorm":0.20901381480858502,"grad-rms/layer_36/moe::expert_fc1":2.1090818052019666e-08,"param-rms/_layer_11_gated_delta::out_norm":0.3273037214546047,"update-rms/_layer_40_gated_delta::out_proj":0.00020465772748122266,"grad-rms/layer_55/moe::shared_experts":1.2356224203584928e-06,"hidden-states-mean/_layer_27_pre_mlp_layernorm":-0.00048335065307923,"grad-absmax/layer_56/moe::expert_fc1":2.493692363714217e-06,"hidden-states-rms/_layer_57_final_layernorm":0.9025098186790947,"hidden-states-mean/_layer_14_residual_attn":7.803590617943182e-05,"hidden-states-absmax/_layer_06_gated_delta::core_attention":0.193359375,"hidden-states-l1/_layer_46_input_layernorm":0.32513404788184735,"hidden-states-mean/_layer_20_pre_mlp_layernorm":-0.00018729861089887301,"hidden-states-mean/_layer_05_residual_attn":0.0018095077077085028,"hidden-states-mean/moe::routed_experts":-0.0001350807073616033,"hidden-states-rms/_layer_37_gated_delta::gate":1.4898733287018473,"param-l1/_layer_51_gated_delta::conv1d":0.04151272401213646,"hidden-states-l1/_layer_33_residual_attn":1.8890945779197905,"hidden-states-rms/_layer_49_residual_attn":10.196960409451487,"hidden-states-rms/_layer_31_residual_attn":5.5187949611553675,"hidden-states-mean/_layer_20_gated_delta::core_attention":0.00029763361727549043,"hidden-states-rms/_layer_13_attention::o_proj":0.2435668833538174,"hidden-states-mean/_layer_56_input_layernorm":-0.0058952102014673,"hidden-states-rms/_layer_12_residual_mlp":1.012620385713124,"grad-absmax/layer_11/gated_delta::A_log":5.410616722656414e-06,"hidden-states-rms/_layer_46_moe::routed_experts":0.2539659869508428,"hidden-states-absmax/_layer_15_gated_delta::g":22.875,"hidden-states-l1/_layer_51_gated_delta::beta":1.460724170294305,"hidden-states-absmax/_layer_51_gated_delta::key":7.4375,"hidden-states-absmax/_layer_07_mlp":8.1875,"hidden-states-mean/_layer_30_gated_delta::query":0.023049241816907144,"grad-absmax/layer_18/moe::expert_fc1":1.6844862784637371e-06,"hidden-states-l1/_layer_25_gated_delta::gate":0.890021072628626,"hidden-states-absmax/_layer_13_residual_mlp":488,"param-l1/_layer_01_gated_delta::out_norm":0.3398590087890625,"param-rms/_layer_38_gated_delta::A_log":1.121809568064354,"grad-rms/layer_04/gated_delta::dt_bias":3.656598490848449e-06,"update-absmax/_layer_27_gated_delta::conv1d":0.00390625,"grad-rms/layer_56/moe::shared_expert_fc1":1.6415897876972346e-07,"hidden-states-mean/_layer_51_residual_mlp":0.12653593737315386,"update-l1/_layer_22_attention::linear_qkv":0.00013033243885729462,"param-mean/_layer_46_gated_delta::conv1d":0.00010075567115563899,"hidden-states-rms/_layer_19_gated_delta::in_proj":1.3189901268807966,"hidden-states-l1/_layer_08_gated_delta::key":0.05328274035844092,"param-absmax/_layer_38_gated_delta::out_proj":0.6171875,"update-l1/_layer_39_gated_delta::A_log":0.0006028390489518642,"hidden-states-absmax/_layer_25_gated_delta::in_proj":17.375,"hidden-states-rms/_layer_54_moe::router":0.07216878508256834,"hidden-states-l1/_layer_43_gated_delta::value":0.0638568179436705,"param-mean/_layer_06_gated_delta::out_norm":0.23595428466796875,"hidden-states-l1/_layer_40_pre_mlp_layernorm":0.157189596556377,"param-l1/_layer_54_gated_delta::out_proj":0.03273315355181694,"hidden-states-mean/_layer_55_moe::router":0.01035015139085474,"hidden-states-l1/_layer_30_gated_delta::beta":1.0934896580564568,"update-mean/_layer_45_gated_delta::A_log":-0.00016620242968201637,"hidden-states-mean/_layer_53_gated_delta::out_proj":-0.0090730033454899,"hidden-states-rms/_layer_21_attention::core_attention":0.3774009404459974,"hidden-states-mean/_layer_55_gated_delta::beta":0.005065646626430348,"hidden-states-rms/_layer_55_moe::router":0.07216878508256834,"hidden-states-absmax/_layer_22_attention::k_layernorm":24.25,"update-rms/_layer_26_gated_delta::A_log":0.000669391591853387,"hidden-states-rms/_layer_22_input_layernorm":0.6856365064924176,"param-mean/_layer_01_gated_delta::conv1d":0.0001273761736229062,"grad-absmax/layer_22/moe::shared_expert_fc2":1.093634455173742e-05,"hidden-states-mean/_layer_41_gated_delta::query":0.05103834209303166,"param-absmax/_layer_17_gated_delta::dt_bias":0.40152162313461304,"hidden-states-l1/_layer_16_gated_delta::beta":1.771003457427437,"update-absmax/_layer_39_gated_delta::A_log":0.0018035173416137695,"moe/layer_13/act_rms_min":0.014941600151360035,"hidden-states-rms/_layer_46_residual_attn":9.036397144728483,"param-rms/_layer_45_gated_delta::A_log":1.2055100914126524,"update-mean/_layer_53_gated_delta::A_log":-0.0002586175687611103,"hidden-states-l1/_layer_06_gated_delta::key":0.05767778994772314,"moe/layer_51/act_rms_min":0.031124357134103775,"params-norm":11140.738517939359,"param-rms/_layer_10_gated_delta::dt_bias":0.19467110221899853,"hidden-states-l1/_layer_44_mlp":0.4101417553593821,"hidden-states-l1/_layer_53_moe::router":0.010322635150573278,"moe/layer_35/act_rms_min_to_median":0.5354552097864994,"hidden-states-l1/_layer_55_gated_delta::in_proj":1.7286472771394779,"hidden-states-l1/_layer_37_gated_delta::out_norm":0.05909606773806426,"hidden-states-absmax/_layer_39_input_layernorm":28.25,"grad-rms/layer_48/moe::shared_expert_fc1":5.951680670871941e-07,"hidden-states-mean/_layer_04_gated_delta::core_attention":1.6777074232781772e-05,"update-l1/_layer_06_gated_delta::out_norm":0.00046539306640625,"moe/layer_22/act_rms_min":0.024750228971242905,"update-l1/_layer_39_gated_delta::conv1d":0.0005731245037168264,"grad-rms/layer_03/moe::expert_fc2":1.2370888237245722e-08,"hidden-states-mean/_layer_05_gated_delta::key":0.013108489734946457,"param-l1/_layer_20_gated_delta::conv1d":0.04006854072213173,"hidden-states-l1/embeddings":0.21960930424968036,"param-rms/_layer_42_gated_delta::conv1d":0.05618036856190076,"grad-rms/layer_48/gated_delta::A_log":3.23832713016371e-05,"hidden-states-rms/_layer_42_moe::router":0.07216878508256834,"hidden-states-l1/_layer_53_moe::routed_experts":0.08981311937321614,"grad-rms/layer_46/moe::expert_fc2":1.917358425163743e-08,"moe/layer_52/act_rms_min_to_median":0.33355675626925846,"grad-absmax/layer_03/moe::expert_fc1":1.263148988073226e-06,"hidden-states-l1/_layer_57_final_layernorm":0.4667140205757642,"moe/layer_02/act_rms_median":0.011371702887117863,"hidden-states-rms/_layer_16_residual_attn":1.2717041958755897,"hidden-states-rms/_layer_17_input_layernorm":0.3070842972374727,"update-l1/_layer_54_gated_delta::conv1d":0.0005493005737662315,"param-l1/_layer_01_gated_delta::dt_bias":0.6410766243934631,"hidden-states-rms/_layer_02_gated_delta::core_attention":0.006022604002936965,"hidden-states-absmax/_layer_33_gated_delta::in_proj":34.5,"hidden-states-l1/_layer_29_moe::routed_experts":0.07519084985785966,"hidden-states-l1/_layer_12_residual_attn":0.3746297210013534,"param-absmax/_layer_12_gated_delta::out_norm":1.234375,"hidden-states-absmax/_layer_30_pre_mlp_layernorm":3.890625,"grad-absmax/layer_38/gated_delta::out_proj":6.888278585392982e-05,"update-rms/_layer_04_gated_delta::conv1d":0.0008465512879842565,"hidden-states-l1/_layer_18_gated_delta::in_proj":0.6609952630874505,"hidden-states-mean/_layer_34_gated_delta::g":-5.681878767237516,"hidden-states-mean/_layer_29_moe::shared_experts":-0.005799964761729808,"param-rms/_layer_25_gated_delta::out_proj":0.04487107088887577,"update-l1/_layer_02_mlp::linear_fc2":0.0001572536857565865,"update-mean/embedding":-1.194131158399614e-07,"hidden-states-absmax/_layer_47_residual_attn":604,"hidden-states-mean/_layer_48_gated_delta::g":-3.191077612766935,"update-l1/_layer_08_gated_delta::out_proj":0.00016252852219622582,"param-l1/_layer_45_gated_delta::conv1d":0.04030940309166908,"moe/layer_40/act_rms_max":0.3537105619907379,"grad-rms/layer_10/gated_delta::out_norm":1.7687865071387586e-05,"hidden-states-mean/_layer_47_attention::core_attention":0.0016983416559060401,"hidden-states-l1/_layer_16_gated_delta::core_attention":0.0006027345216367569,"update-l1/_layer_05_gated_delta::out_norm":0.0006103515625,"hidden-states-rms/_layer_05_mlp":0.06467084910587004,"update-l1/_layer_18_gated_delta::conv1d":0.0006297594518400729,"param-mean/_layer_16_gated_delta::out_proj":1.270762732019648e-05,"hidden-states-mean/_layer_18_gated_delta::out_norm":-0.00044575542635907846,"hidden-states-absmax/_layer_38_input_layernorm":29,"param-l1/_layer_12_gated_delta::A_log":1.0383905172348022,"grad-rms/layer_25/moe::expert_fc2":2.1725472164451605e-08,"hidden-states-rms/_layer_52_mlp":1.1805871927451956,"hidden-states-absmax/_layer_41_residual_attn":502,"hidden-states-l1/_layer_54_gated_delta::out_proj":1.0950170868655547,"hidden-states-absmax/_layer_35_pre_mlp_layernorm":4.15625,"param-absmax/lm_head":1.6953125,"hidden-states-rms/_layer_10_gated_delta::in_proj":1.2022722389477658,"hidden-states-l1/_layer_27_gated_delta::g":7.3365325136819495,"hidden-states-l1/_layer_28_moe::routed_experts":0.06114532865446803,"hidden-states-l1/_layer_05_gated_delta::gate":0.7450055772072764,"hidden-states-absmax/_layer_44_gated_delta::gate":24.5,"update-absmax/_layer_50_attention::linear_qkv":0.00390625,"hidden-states-l1/_layer_41_gated_delta::beta":1.3325746455877596,"hidden-states-l1/_layer_12_gated_delta::core_attention":0.0015267455674926123,"param-mean/_layer_07_gated_delta::A_log":1.144558310508728,"grad-rms/layer_03/gated_delta::conv1d":1.1208229934976917e-05,"update-rms/_layer_52_gated_delta::out_norm":0.0018911051495153403,"hidden-states-absmax/_layer_37_moe::routed_experts":24,"param-l1/_layer_35_gated_delta::conv1d":0.038013115525245667,"grad-absmax/layer_09/gated_delta::A_log":7.414100309688365e-06,"param-l1/_layer_47_attention::o_proj":0.04212373122572899,"hidden-states-absmax/_layer_22_attention::o_proj":15.875,"hidden-states-l1/_layer_23_gated_delta::core_attention":0.0011194758287564907,"hidden-states-l1/_layer_37_input_layernorm":0.23931281141195435,"hidden-states-mean/_layer_16_gated_delta::value":0.0025268353740078658,"hidden-states-l1/_layer_39_gated_delta::out_norm":0.08568173923274747,"hidden-states-rms/_layer_25_gated_delta::out_proj":0.4401997230215801,"hidden-states-l1/_layer_27_gated_delta::out_proj":0.20854255062848934,"param-mean/_layer_29_gated_delta::dt_bias":0.14726287126541138,"hidden-states-absmax/_layer_10_gated_delta::g":24.25,"update-absmax/_layer_03_gated_delta::out_norm":0.001953125,"hidden-states-mean/_layer_26_residual_mlp":-0.018980628279522453,"hidden-states-l1/_layer_07_residual_attn":0.2482315299786961,"grad-rms/layer_07/gated_delta::out_proj":2.883142507287327e-07,"hidden-states-mean/_layer_24_moe::router":0.01023259052357523,"hidden-states-rms/_layer_38_moe::router":0.07216878508256834,"grad-absmax/layer_12/gated_delta::A_log":0.00029073309269733727,"param-l1/_layer_29_gated_delta::out_proj":0.033908914774656296,"hidden-states-rms/_layer_44_mlp":0.6300259825227432,"hidden-states-mean/_layer_08_moe::routed_experts":-7.4186408759365175e-06,"hidden-states-l1/_layer_54_gated_delta::g":5.240720419969609,"param-l1/_layer_41_gated_delta::A_log":0.9695128798484802,"hidden-states-absmax/_layer_15_moe::router":0.9994155168533325,"update-mean/_layer_41_gated_delta::out_norm":6.103515625e-05,"hidden-states-absmax/_layer_41_moe::routed_experts":30.75,"hidden-states-absmax/_layer_40_moe::router":0.99969083070755,"hidden-states-rms/_layer_12_moe::router":0.07216878508256834,"hidden-states-absmax/_layer_19_gated_delta::value":1.6015625,"hidden-states-mean/_layer_54_input_layernorm":0.002683092228542832,"hidden-states-rms/_layer_29_moe::router":0.07216878508256834,"hidden-states-absmax/_layer_17_gated_delta::g":20.625,"update-l1/_layer_49_attention::o_proj":0.00018990050011780113,"hidden-states-rms/_layer_45_gated_delta::core_attention":0.011406851937135427,"param-l1/_layer_55_gated_delta::A_log":1.707268476486206,"hidden-states-l1/_layer_56_gated_delta::query":0.09667183121106997,"param-absmax/_layer_26_gated_delta::out_norm":1.75,"hidden-states-absmax/_layer_52_gated_delta::g":18.875,"hidden-states-absmax/_layer_08_gated_delta::in_proj":20.875,"hidden-states-absmax/_layer_19_moe::shared_experts":8.625,"grad-rms/layer_41/gated_delta::out_proj":3.5726607475873894e-07,"hidden-states-l1/_layer_20_moe::routed_experts":0.038151357461608824,"grad-rms/layer_56/gated_delta::A_log":6.737279426220121e-06,"param-mean/_layer_08_gated_delta::out_proj":-1.715215330477804e-05,"update-absmax/_layer_23_gated_delta::out_proj":0.00390625,"grad-rms/layer_40/moe::shared_experts":1.891209523720272e-06,"param-l1/_layer_19_gated_delta::A_log":1.167241096496582,"hidden-states-l1/_layer_53_mlp":0.666169186373852,"hidden-states-l1/_layer_44_gated_delta::gate":1.3278230819491246,"hidden-states-l1/_layer_04_moe::router":0.01035224810874432,"grad-absmax/layer_43/gated_delta::out_norm":4.3102609197376296e-05,"update-l1/_layer_20_gated_delta::conv1d":0.000612124043982476,"hidden-states-l1/_layer_33_gated_delta::key":0.13116718902196753,"param-rms/_layer_48_gated_delta::conv1d":0.04455283952931131,"update-mean/_layer_01_gated_delta::out_proj":-3.5204489989837384e-08,"update-mean/_layer_32_gated_delta::out_proj":7.621008535352303e-08,"update-l1/_layer_04_gated_delta::conv1d":0.0006663469830527902,"hidden-states-mean/_layer_37_gated_delta::core_attention":-4.910346109326279e-05,"update-l1/_layer_25_gated_delta::conv1d":0.0006423700251616538,"hidden-states-l1/_layer_52_gated_delta::out_proj":0.81673178872249,"hidden-states-l1/_layer_55_gated_delta::value":0.06098570656606037,"update-absmax/_layer_05_gated_delta::out_proj":0.00390625,"hidden-states-absmax/_layer_18_residual_attn":494,"hidden-states-rms/_layer_17_gated_delta::beta":3.27176368007432,"moe/layer_37/act_rms_max":0.30205070972442627,"hidden-states-absmax/_layer_52_input_layernorm":30,"param-absmax/_layer_40_gated_delta::dt_bias":0.6012575626373291,"grad-absmax/layer_14/gated_delta::out_norm":6.299537926679477e-05,"hidden-states-l1/_layer_35_residual_mlp":2.1214668895562756,"hidden-states-mean/_layer_54_residual_mlp":0.08067568608232571,"hidden-states-l1/_layer_31_gated_delta::core_attention":0.00048388866655318935,"hidden-states-rms/_layer_44_gated_delta::key":0.17343204853277075,"param-mean/_layer_22_attention::o_proj":-1.843120298872236e-05,"grad-absmax/layer_29/gated_delta::dt_bias":5.8406453717907425e-06,"update-rms/_layer_45_gated_delta::out_proj":0.00020279648924592924,"param-absmax/_layer_13_attention::linear_qkv":0.56640625,"hidden-states-l1/_layer_43_gated_delta::core_attention":0.000590388209623928,"hidden-states-absmax/_layer_49_residual_mlp":616,"update-mean/_layer_36_gated_delta::conv1d":7.160851964727044e-07,"grad-rms/layer_39/moe::router":3.412927958659581e-07,"hidden-states-absmax/_layer_19_input_layernorm":18.75,"update-mean/_layer_19_gated_delta::conv1d":2.7151165340910666e-05,"hidden-states-absmax/_layer_35_gated_delta::key":6.875,"param-l1/_layer_33_gated_delta::out_proj":0.034766796976327896,"grad-absmax/layer_21/moe::shared_expert_fc2":1.1465229363238905e-05,"update-rms/_layer_23_gated_delta::A_log":0.0006911198632155799,"hidden-states-mean/_layer_28_residual_mlp":-0.022013990970133658,"hidden-states-mean/_layer_44_gated_delta::out_proj":-0.002690820809849597,"grad-rms/layer_52/moe::shared_expert_fc2":4.2541306216873355e-07,"update-mean/_layer_32_gated_delta::dt_bias":-1.1017953511327505e-05,"grad-rms/layer_05/moe::shared_expert_fc2":1.3339819028985146e-07,"hidden-states-mean/_layer_25_moe::routed_experts":-0.00016420310461695616,"hidden-states-l1/_layer_15_mlp":0.09104676884970293,"grad-absmax/layer_42/gated_delta::out_norm":2.1149329768377356e-05,"hidden-states-mean/_layer_26_gated_delta::g":-4.8985916255225685,"hidden-states-mean/_layer_49_moe::shared_experts":-0.002198161819358089,"grad-rms/layer_06/moe::router":4.193126586606296e-07,"hidden-states-absmax/_layer_31_input_layernorm":29.25,"hidden-states-mean/_layer_55_gated_delta::g":-3.736746041386848,"update-l1/_layer_32_gated_delta::dt_bias":0.000619507918599993,"hidden-states-absmax/_layer_20_gated_delta::g":15.6875,"hidden-states-absmax/_layer_45_gated_delta::out_norm":103.5,"hidden-states-absmax/_layer_54_gated_delta::beta":12.75,"param-mean/_layer_18_gated_delta::A_log":0.764329195022583,"update-absmax/_layer_39_gated_delta::out_proj":0.00390625,"hidden-states-l1/_layer_15_moe::routed_experts":0.029832028818527204,"param-rms/_layer_48_gated_delta::out_proj":0.04174495986768549,"update-mean/_layer_54_gated_delta::out_norm":0,"hidden-states-absmax/_layer_18_input_layernorm":15.25,"param-absmax/_layer_25_gated_delta::out_proj":1.21875,"param-rms/_layer_23_gated_delta::A_log":1.0376757208685152,"grad-rms/layer_38/moe::shared_experts":1.813699484620872e-06,"hidden-states-mean/_layer_23_gated_delta::key":0.02821541541402524,"grad-absmax/layer_11/gated_delta::out_proj":3.574465881683864e-05,"update-mean/_layer_31_gated_delta::out_norm":-0.000244140625,"hidden-states-mean/_layer_08_gated_delta::beta":0.41487358985824463,"hidden-states-rms/_layer_41_moe::routed_experts":0.18741408804704449,"update-l1/_layer_15_gated_delta::out_proj":0.0001468666596338153,"hidden-states-mean/_layer_36_mlp":-0.003443724187832974,"param-mean/_layer_37_gated_delta::out_proj":-3.541720434441231e-05,"hidden-states-absmax/_layer_11_mlp":12.3125,"update-absmax/_layer_42_gated_delta::A_log":0.0016362667083740234,"update-rms/_layer_37_gated_delta::conv1d":0.0008025593641583906,"update-absmax/_layer_56_gated_delta::A_log":0.0016465187072753906,"update-mean/_layer_39_gated_delta::dt_bias":-0.0002558025298640132,"hidden-states-absmax/_layer_18_gated_delta::gate":8.25,"grad-absmax/layer_37/moe::shared_expert_fc1":8.24610106064938e-06,"hidden-states-rms/_layer_25_pre_mlp_layernorm":0.17088749370391318,"hidden-states-absmax/_layer_16_residual_mlp":494,"update-absmax/_layer_43_gated_delta::conv1d":0.00390625,"hidden-states-rms/_layer_05_residual_attn":0.5946286479940724,"hidden-states-l1/_layer_35_mlp":0.2503818273999485,"param-rms/gated_delta::A_log":1.2925474747236012,"hidden-states-mean/_layer_13_residual_mlp":-0.0005562117520674172,"hidden-states-l1/_layer_56_gated_delta::out_proj":1.5346208256031066,"hidden-states-l1/_layer_14_gated_delta::key":0.07278435054830916,"hidden-states-mean/_layer_31_input_layernorm":0.0016569865048809654,"hidden-states-absmax/_layer_19_moe::router":0.9991490244865417,"grad-rms/layer_22/attention::linear_qkv":2.085436648152007e-06,"grad-absmax/layer_17/gated_delta::out_norm":4.105858533876017e-05,"param-rms/_layer_44_gated_delta::conv1d":0.06391220620549631,"grad-absmax/layer_08/moe::shared_expert_fc2":1.5584998891426949e-06,"hidden-states-rms/_layer_06_gated_delta::value":0.07014313226953904,"hidden-states-rms/_layer_39_gated_delta::core_attention":0.0021124901294906754,"param-l1/_layer_16_gated_delta::out_proj":0.03399470075964928,"grad-rms/layer_47/moe::shared_expert_fc1":3.545055192625916e-07,"hidden-states-rms/_layer_32_mlp":0.3582910699530043,"hidden-states-mean/_layer_45_mlp":-0.0003700173260757865,"hidden-states-absmax/_layer_46_residual_mlp":604,"hidden-states-l1/_layer_51_gated_delta::key":0.1849401068616041,"moe/layer_02/act_rms_min_to_median":0.25333770422717933,"hidden-states-l1/_layer_21_attention::o_proj":0.22830742338984292,"hidden-states-absmax/_layer_08_gated_delta::g":13.1875,"hidden-states-rms/_layer_52_pre_mlp_layernorm":0.25420634851314444,"grad-rms/layer_23/moe::expert_fc2":2.1127781797160036e-08,"param-mean/_layer_36_gated_delta::out_norm":1.0648193359375,"grad-rms/layer_42/moe::shared_expert_fc1":2.3156445282295044e-07,"param-rms/_layer_39_gated_delta::out_norm":1.1030121051196706,"moe/layer_11/act_rms_min_to_median":0.28685009829256775,"hidden-states-rms/_layer_19_gated_delta::gate":1.1611174816505523,"hidden-states-mean/_layer_09_moe::routed_experts":-2.920183463921028e-05,"update-absmax/_layer_17_gated_delta::out_proj":0.00390625,"hidden-states-mean/_layer_20_gated_delta::g":-5.164252419978669,"hidden-states-rms/_layer_33_gated_delta::query":0.6773363826913857,"hidden-states-l1/_layer_49_residual_mlp":4.123186602972341,"grad-absmax/layer_21/moe::expert_fc1":4.950967195327394e-06,"update-mean/_layer_48_gated_delta::A_log":0.00017452239990234375,"update-l1/_layer_38_gated_delta::conv1d":0.0005903849960304797,"update-l1/_layer_42_gated_delta::A_log":0.00042319876956753433,"update-rms/_layer_16_gated_delta::A_log":0.0006620246439605038,"grad-absmax/layer_20/gated_delta::A_log":1.3714315173274372e-05,"param-rms/_layer_28_gated_delta::out_proj":0.043355576890122656,"update-rms/_layer_01_gated_delta::out_norm":0.0007720404443770458,"hidden-states-absmax/_layer_36_moe::routed_experts":25.75,"grad-rms/layer_17/gated_delta::dt_bias":2.869495005780224e-06,"hidden-states-mean/_layer_29_gated_delta::beta":-2.2473585900495756,"hidden-states-rms/_layer_42_gated_delta::key":0.10277733753025335,"hidden-states-absmax/_layer_26_gated_delta::core_attention":0.0947265625,"hidden-states-absmax/_layer_17_gated_delta::in_proj":20.625,"update-rms/_layer_11_gated_delta::out_proj":0.00021120488504955677,"update-rms/_layer_21_attention::linear_qkv":0.00018299903421200927,"update-absmax/_layer_44_gated_delta::out_norm":0.0078125,"update-l1/embedding":0.001145453774370253,"hidden-states-absmax/_layer_08_pre_mlp_layernorm":2.078125,"grad-rms/layer_36/gated_delta::out_norm":4.5572621079496904e-05,"hidden-states-absmax/_layer_19_gated_delta::query":6.625,"param-mean/attention::linear_qkv":0.00013381108874455094,"moe/layer_52/act_rms_max":0.7487149834632874,"hidden-states-rms/_layer_25_gated_delta::out_norm":0.17033767173478861,"update-absmax/_layer_34_gated_delta::A_log":0.0014677047729492188,"update-absmax/_layer_37_gated_delta::out_norm":0.0078125,"update-l1/_layer_54_gated_delta::dt_bias":0.00043635303154587746,"grad-rms/layer_44/moe::router":7.892978150852807e-07,"param-absmax/_layer_08_gated_delta::conv1d":0.392578125,"hidden-states-absmax/_layer_49_attention::core_attention":18.875,"update-rms/_layer_39_gated_delta::A_log":0.0007498582702815024,"param-l1/_layer_43_gated_delta::A_log":1.0343066453933716,"hidden-states-l1/_layer_14_input_layernorm":0.1681942991249918,"hidden-states-l1/_layer_28_input_layernorm":0.20498020410301734,"hidden-states-absmax/_layer_50_residual_mlp":740,"param-rms/_layer_37_gated_delta::out_norm":1.2653359448047885,"grad-absmax/layer_28/moe::shared_expert_fc2":2.3559407509310404e-06,"param-l1/_layer_26_gated_delta::out_norm":0.938232421875,"hidden-states-mean/_layer_09_gated_delta::in_proj":-0.19644437685784016,"hidden-states-l1/_layer_18_gated_delta::key":0.06334713856543762,"hidden-states-mean/_layer_08_moe::router":0.01024271089716131,"update-l1/_layer_53_gated_delta::out_norm":0.00042724609375,"hidden-states-rms/_layer_44_residual_mlp":8.382773409187136,"hidden-states-mean/_layer_16_gated_delta::out_proj":-0.00036934598793861127,"grad-rms/layer_25/gated_delta::A_log":1.5258768286216261e-06,"hidden-states-rms/_layer_09_gated_delta::gate":1.33044642566168,"hidden-states-rms/_layer_55_moe::routed_experts":0.21096344290027297,"grad-absmax/layer_15/moe::expert_fc1":1.3634546576213324e-06,"hidden-states-absmax/_layer_44_mlp":66.5,"hidden-states-rms/_layer_02_gated_delta::in_proj":1.2014426881347189,"grad-rms/layer_16/moe::shared_experts":9.044137249994368e-07,"update-absmax/_layer_19_gated_delta::out_proj":0.00390625,"grad-absmax/layer_33/gated_delta::out_proj":7.790820745867677e-06,"param-rms/_layer_43_gated_delta::out_norm":0.857921967183568,"param-mean/_layer_44_gated_delta::dt_bias":0.5241995453834534,"hidden-states-mean/_layer_54_gated_delta::in_proj":-0.19429225161444125,"param-absmax/_layer_54_gated_delta::conv1d":0.443359375,"grad-rms/layer_17/gated_delta::conv1d":1.8671980487521775e-05,"hidden-states-rms/_layer_51_gated_delta::value":0.17649391661881084,"hidden-states-absmax/_layer_38_gated_delta::g":11.875,"hidden-states-mean/_layer_16_gated_delta::key":0.017029883507476272,"hidden-states-mean/_layer_25_gated_delta::key":0.009395885470019039,"grad-rms/layer_15/moe::expert_fc1":2.0515051746662665e-08,"param-absmax/_layer_37_gated_delta::conv1d":0.53515625,"grad-absmax/layer_03/moe::shared_expert_fc1":3.974214268964715e-05,"hidden-states-mean/_layer_28_gated_delta::out_norm":0.0012856041216825391,"update-rms/_layer_56_gated_delta::conv1d":0.0006435806047696037,"hidden-states-absmax/_layer_56_gated_delta::beta":17.375,"hidden-states-l1/_layer_33_mlp":0.22139837996940373,"hidden-states-absmax/_layer_40_input_layernorm":33.5,"param-absmax/_layer_51_gated_delta::dt_bias":0.7043432593345642,"grad-rms/layer_41/moe::expert_fc2":1.9349078472606822e-08,"update-l1/mlp::linear_fc2":0.00015632598660886288,"hidden-states-l1/_layer_09_gated_delta::out_norm":0.02170635273532304,"hidden-states-l1/_layer_45_mlp":0.49686962328787126,"update-absmax/_layer_45_gated_delta::out_norm":0.00390625,"hidden-states-absmax/_layer_06_gated_delta::g":26.25,"hidden-states-rms/_layer_31_mlp":0.34538108468884426,"update-l1/attention::o_proj":0.00015230011610713388,"param-mean/_layer_18_gated_delta::dt_bias":0.03628501668572426,"update-absmax/_layer_06_gated_delta::conv1d":0.0031280517578125,"hidden-states-mean/_layer_35_residual_mlp":-0.056925042799120286,"hidden-states-absmax/_layer_29_moe::shared_experts":103.5,"grad-absmax/layer_53/gated_delta::out_proj":0.00010819085582625121,"hidden-states-rms/_layer_42_mlp":0.5058684778642478,"hidden-states-absmax/_layer_36_gated_delta::core_attention":0.29296875,"update-l1/_layer_03_gated_delta::dt_bias":0.0006163166253827512,"hidden-states-l1/_layer_02_gated_delta::beta":1.411276031884674,"hidden-states-rms/_layer_43_mlp":0.5770943277661387,"hidden-states-absmax/_layer_14_input_layernorm":18.125,"hidden-states-mean/_layer_49_moe::router":0.01025349064723502,"hidden-states-rms/_layer_20_gated_delta::g":5.845812124386584,"param-absmax/_layer_09_gated_delta::conv1d":0.65234375,"hidden-states-l1/_layer_39_input_layernorm":0.24588316629924373,"hidden-states-absmax/_layer_05_moe::shared_experts":7.21875,"hidden-states-mean/_layer_38_moe::shared_experts":-0.0017338607897369016,"hidden-states-absmax/_layer_25_moe::router":0.9954990148544312,"param-l1/_layer_26_gated_delta::dt_bias":0.1266481727361679,"grad-rms/layer_23/gated_delta::dt_bias":4.6547852595378395e-06,"grad-absmax/layer_25/gated_delta::A_log":3.697080501297023e-06,"grad-absmax/layer_23/moe::shared_experts":2.106637657561805e-05,"hidden-states-absmax/_layer_52_moe::router":0.9998927712440491,"update-l1/_layer_41_gated_delta::A_log":0.0004924435634166002,"hidden-states-rms/_layer_18_gated_delta::key":0.12786414415775632,"grad-absmax/layer_37/moe::expert_fc2":3.4074464565492235e-06,"grad-rms/layer_18/gated_delta::out_proj":4.358819378009876e-07,"hidden-states-l1/_layer_15_gated_delta::value":0.028939971424933843,"param-l1/_layer_28_gated_delta::dt_bias":0.19991391897201538,"grad-rms/layer_44/moe::shared_expert_fc2":2.061009243006357e-07,"hidden-states-l1/_layer_53_gated_delta::out_proj":0.8789843772631142,"param-absmax/_layer_49_attention::o_proj":0.427734375,"hidden-states-rms/_layer_47_moe::shared_experts":0.5578405529401038,"hidden-states-absmax/_layer_22_moe::routed_experts":11.6875,"hidden-states-mean/_layer_33_moe::shared_experts":-0.00190544954398418,"hidden-states-mean/_layer_18_moe::router":0.010264109588918621,"grad-absmax/layer_29/moe::expert_fc1":1.0895155355683528e-05,"hidden-states-mean/_layer_38_gated_delta::g":-2.088835488418995,"update-rms/_layer_04_gated_delta::dt_bias":0.00039056269027775206,"update-rms/_layer_20_gated_delta::out_norm":0.0019983771415704225,"hidden-states-l1/_layer_20_mlp":0.1391828971127149,"moe/layer_47/act_rms_min":0.0768054649233818,"grad-absmax/layer_26/gated_delta::dt_bias":5.331342890713131e-06,"hidden-states-mean/_layer_07_gated_delta::out_norm":8.92424502100643e-07,"hidden-states-l1/_layer_32_moe::routed_experts":0.07506416122965628,"hidden-states-l1/_layer_06_gated_delta::out_norm":0.015677066526063266,"update-mean/_layer_41_gated_delta::conv1d":-3.270824663559324e-06,"update-absmax/_layer_52_gated_delta::out_proj":0.0078125,"update-l1/_layer_41_gated_delta::out_proj":0.00014260098396334797,"hidden-states-l1/_layer_28_gated_delta::out_proj":0.25728154269236836,"hidden-states-mean/_layer_05_gated_delta::out_norm":-0.0007283449808397866,"hidden-states-mean/_layer_33_pre_mlp_layernorm":0.000127734253276125,"update-rms/_layer_40_gated_delta::A_log":0.0007043673726826587,"hidden-states-rms/_layer_56_gated_delta::g":5.580256198040607,"param-rms/_layer_05_gated_delta::conv1d":0.05771927356663225,"hidden-states-l1/_layer_40_gated_delta::in_proj":1.3140039621992665,"grad-rms/layer_44/moe::shared_expert_fc1":3.2229257404410165e-07,"hidden-states-l1/_layer_25_gated_delta::value":0.03261591137837307,"hidden-states-mean/_layer_35_pre_mlp_layernorm":0.00015800331740120857,"grad-absmax/layer_37/gated_delta::dt_bias":7.683550211368129e-05,"hidden-states-mean/_layer_19_gated_delta::core_attention":7.447070835135008e-05,"update-absmax/_layer_34_gated_delta::conv1d":0.0029296875,"hidden-states-mean/_layer_35_gated_delta::query":0.03199536734764187,"hidden-states-rms/_layer_25_moe::routed_experts":0.11264018162657173,"param-absmax/_layer_05_gated_delta::out_norm":0.6640625,"hidden-states-absmax/_layer_34_pre_mlp_layernorm":4.03125,"hidden-states-absmax/_layer_12_gated_delta::gate":22.625,"param-rms/_layer_03_gated_delta::conv1d":0.05494369374600923,"hidden-states-absmax/_layer_20_moe::shared_experts":59.75,"moe/layer_42/act_rms_max":0.4676974415779114,"_step":2384,"update-absmax/_layer_44_gated_delta::conv1d":0.0029296875,"param-mean/_layer_09_gated_delta::conv1d":0.0005815323092974722,"param-rms/_layer_37_gated_delta::conv1d":0.055760467435088105,"hidden-states-l1/mlp":0.24244394894819418,"hidden-states-l1/_layer_34_moe::router":0.010208949827583604,"grad-absmax/layer_28/gated_delta::conv1d":0.0009047158528119326,"param-absmax/_layer_39_gated_delta::out_norm":1.7421875,"update-mean/_layer_15_gated_delta::A_log":0.0001680204295553267,"param-mean/_layer_35_gated_delta::out_proj":-2.7588654120336287e-05,"hidden-states-absmax/_layer_19_residual_mlp":494,"hidden-states-l1/_layer_05_mlp":0.04139182837508063,"grad-rms/layer_37/moe::expert_fc1":2.029403059303769e-08,"param-mean/_layer_31_gated_delta::out_norm":0.911224365234375,"grad-absmax/layer_41/gated_delta::dt_bias":8.722300117369741e-05,"param-rms/_layer_55_gated_delta::conv1d":0.046149982383104464,"update-absmax/_layer_56_gated_delta::conv1d":0.003021240234375,"hidden-states-absmax/_layer_08_residual_attn":59,"hidden-states-mean/_layer_54_gated_delta::gate":-0.3831428150108832,"hidden-states-mean/_layer_28_residual_attn":-0.021268642248945504,"hidden-states-l1/_layer_34_gated_delta::beta":1.6377633378223584,"hidden-states-rms/_layer_21_residual_attn":1.6166815445850042,"update-rms/_layer_30_gated_delta::A_log":0.000521072081847242,"update-rms/_layer_29_gated_delta::out_proj":0.00020475949707840995,"param-mean/_layer_53_gated_delta::conv1d":-5.3192226914688945e-05,"hidden-states-absmax/_layer_15_input_layernorm":13.125,"hidden-states-l1/_layer_31_moe::routed_experts":0.0745348488762563,"hidden-states-l1/_layer_50_moe::routed_experts":0.1415025985920705,"update-absmax/_layer_26_gated_delta::out_proj":0.00390625,"param-mean/_layer_37_gated_delta::dt_bias":0.4097752869129181,"grad-absmax/layer_24/moe::expert_fc1":4.496428118727636e-06,"hidden-states-mean/_layer_47_moe::router":0.01022758497438192,"param-mean/_layer_51_gated_delta::out_norm":0.993896484375,"hidden-states-mean/_layer_04_mlp":0.00021312004167527458,"hidden-states-l1/_layer_10_gated_delta::g":5.652248102694523,"hidden-states-l1/_layer_49_moe::routed_experts":0.17325997579913616,"param-mean/_layer_17_gated_delta::dt_bias":-0.017835063859820366,"param-mean/_layer_15_gated_delta::out_proj":-5.077305104350671e-06,"hidden-states-rms/_layer_15_gated_delta::beta":2.8892041884421658,"hidden-states-absmax/_layer_48_gated_delta::beta":11.5,"hidden-states-mean/_layer_56_gated_delta::out_norm":-0.01063760496826231,"hidden-states-mean/_layer_23_gated_delta::in_proj":0.04260810028905989,"hidden-states-l1/_layer_34_input_layernorm":0.30529753032893303,"param-absmax/_layer_45_gated_delta::out_proj":1.734375,"param-l1/_layer_07_gated_delta::out_proj":0.035652413964271545,"hidden-states-rms/_layer_30_moe::routed_experts":0.13971247074893622,"hidden-states-absmax/_layer_19_gated_delta::beta":12.9375,"hidden-states-rms/_layer_02_residual_mlp":0.5308526496157516,"grad-rms/layer_37/gated_delta::conv1d":3.0856908533506915e-05,"grad-absmax/layer_31/moe::shared_expert_fc1":8.61129319673637e-06,"hidden-states-rms/_layer_34_gated_delta::beta":2.042758097037581,"hidden-states-l1/_layer_10_gated_delta::value":0.02516844423633149,"moe/layer_15/act_rms_max":0.09316903352737427,"hidden-states-l1/_layer_39_gated_delta::g":3.662489850817901,"update-absmax/_layer_02_mlp::linear_fc2":0.001953125,"grad-rms/layer_22/moe::shared_experts":1.1951218938121575e-06,"hidden-states-absmax/_layer_24_attention::core_attention":9.5,"update-l1/_layer_33_gated_delta::out_proj":0.0001474805030738935,"hidden-states-absmax/_layer_14_moe::router":0.9999935626983643,"hidden-states-mean/_layer_53_moe::shared_experts":-0.015508392811822034,"update-absmax/_layer_45_gated_delta::dt_bias":0.001524999737739563,"hidden-states-rms/_layer_54_gated_delta::core_attention":0.007790900241767721,"moe/layer_00/act_rms_max":0,"hidden-states-l1/_layer_20_gated_delta::query":0.06849268895300495,"hidden-states-absmax/_layer_17_mlp":5.46875,"hidden-states-mean/_layer_15_moe::routed_experts":3.2661969872954956e-05,"hidden-states-l1/_layer_32_gated_delta::core_attention":0.004591369617446615,"update-rms/_layer_38_gated_delta::out_norm":0.0015440808887540916,"param-rms/_layer_03_gated_delta::out_norm":0.14094327962363068,"update-rms/_layer_02_mlp::linear_fc2":0.0002220967363428337,"hidden-states-absmax/_layer_54_residual_attn":2768,"param-rms/_layer_56_gated_delta::out_norm":1.3948960454894723,"hidden-states-mean/_layer_57_final_layernorm":0.0009165148877668247,"param-rms/_layer_52_gated_delta::out_norm":0.9045855265385346,"hidden-states-l1/_layer_44_input_layernorm":0.29934123746648283,"grad-rms/layer_19/gated_delta::out_proj":5.445414440340616e-07,"update-absmax/_layer_03_gated_delta::conv1d":0.002960205078125,"hidden-states-rms/_layer_06_gated_delta::gate":1.3718206704181535,"param-mean/_layer_02_gated_delta::out_proj":-5.1083570724586025e-05,"grad-absmax/moe::router":0.00016218451492022723,"hidden-states-rms/_layer_07_moe::routed_experts":0.029517838496064288,"update-l1/_layer_23_gated_delta::out_proj":0.0001376930740661919,"hidden-states-rms/_layer_21_attention::k_layernorm":3.63101179855461,"hidden-states-rms/_layer_11_mlp":0.11082277953108335,"hidden-states-l1/_layer_35_moe::router":0.010168227431095033,"param-rms/_layer_01_gated_delta::out_proj":0.03986448661116354,"hidden-states-absmax/_layer_16_gated_delta::g":17.75,"hidden-states-mean/_layer_31_moe::router":0.01021219331418567,"update-absmax/_layer_53_gated_delta::out_norm":0.0078125,"param-absmax/_layer_16_gated_delta::out_proj":0.90625,"hidden-states-rms/_layer_34_moe::routed_experts":0.15608556964130932,"param-rms/_layer_48_gated_delta::dt_bias":0.7257267236501932,"param-rms/_layer_03_gated_delta::dt_bias":0.46666223986546923,"moe/layer_33/act_rms_median":0.12529237568378448,"hidden-states-mean/_layer_38_gated_delta::value":-0.0018733936499751986,"hidden-states-rms/_layer_34_pre_mlp_layernorm":0.20037457218528132,"update-absmax/gated_delta::out_norm":0.0078125,"grad-rms/layer_39/gated_delta::out_proj":3.8698985030749857e-07,"hidden-states-absmax/_layer_49_attention::q_layernorm":21.625,"param-absmax/_layer_41_gated_delta::out_norm":1.7890625,"hidden-states-rms/_layer_54_mlp":0.5267608560427672,"param-l1/_layer_40_gated_delta::conv1d":0.0391806960105896,"hidden-states-mean/_layer_06_moe::router":0.01027064126561344,"hidden-states-absmax/_layer_49_residual_attn":616,"update-mean/_layer_31_gated_delta::conv1d":3.328450247863657e-06,"update-l1/_layer_31_gated_delta::A_log":0.0006316141225397587,"update-rms/_layer_54_gated_delta::dt_bias":0.0005293628065867922,"hidden-states-absmax/_layer_39_gated_delta::value":6.90625,"hidden-states-rms/_layer_41_gated_delta::beta":1.7741722281469607,"hidden-states-mean/_layer_13_residual_attn":-0.001207944498971939,"param-rms/_layer_22_attention::o_proj":0.039035740269560186,"grad-absmax/layer_04/moe::expert_fc1":8.130853075272171e-07,"param-l1/_layer_39_gated_delta::out_norm":1.06982421875,"hidden-states-mean/_layer_41_gated_delta::beta":-0.7448810034848455,"grad-absmax/layer_46/gated_delta::out_norm":5.1075363444397226e-05,"hidden-states-l1/_layer_51_gated_delta::in_proj":2.1464152532898,"param-l1/_layer_30_gated_delta::out_norm":0.734710693359375,"hidden-states-absmax/_layer_51_gated_delta::query":44.75,"hidden-states-absmax/_layer_38_moe::routed_experts":24.5,"update-rms/_layer_19_gated_delta::dt_bias":0.00047953582581913823,"grad-absmax/layer_03/gated_delta::A_log":2.911132287408691e-05,"param-rms/gated_delta::dt_bias":0.4315508043511444,"hidden-states-absmax/_layer_08_residual_mlp":61,"param-rms/_layer_28_gated_delta::dt_bias":0.23494118808054706,"param-absmax/_layer_18_gated_delta::A_log":1.7932859659194946,"param-l1/_layer_20_gated_delta::A_log":1.14625084400177,"update-mean/_layer_26_gated_delta::out_norm":-0.00054931640625,"hidden-states-absmax/_layer_01_gated_delta::key":3.671875,"hidden-states-rms/_layer_01_pre_mlp_layernorm":0.07807230829622101,"hidden-states-rms/_layer_36_gated_delta::out_proj":0.787820849123719,"hidden-states-mean/_layer_44_gated_delta::key":0.031556766516776864,"update-l1/_layer_10_gated_delta::conv1d":0.0006538224406540394,"hidden-states-rms/_layer_23_residual_attn":1.8854784173680113,"grad-rms/layer_17/gated_delta::A_log":2.9061377378273403e-06,"grad-absmax/layer_27/gated_delta::out_proj":2.6241843443131074e-05,"update-rms/_layer_43_gated_delta::A_log":0.0006224347045638634,"hidden-states-rms/gated_delta::query":0.3929732378390102,"update-l1/gated_delta::out_norm":0.0005664436184630102,"update-l1/_layer_39_gated_delta::dt_bias":0.0006086560315452516,"hidden-states-mean/_layer_33_gated_delta::g":-5.7753550738519515,"grad-rms/layer_48/moe::shared_expert_fc2":3.503289183059746e-07,"grad-absmax/embedding":1.1172659469593782e-05,"update-l1/_layer_29_gated_delta::out_norm":0.0007476806640625,"hidden-states-rms/_layer_46_gated_delta::query":0.234229048390755,"hidden-states-absmax/_layer_45_gated_delta::value":11.625,"hidden-states-l1/_layer_46_gated_delta::key":0.10570872345074694,"grad-rms/layer_12/moe::router":4.7355433665821973e-07,"hidden-states-rms/_layer_26_mlp":0.25615023394768466,"grad-absmax/layer_17/moe::expert_fc1":1.5717459973529913e-06,"hidden-states-l1/_layer_07_gated_delta::query":0.05094820123293414,"hidden-states-l1/_layer_29_gated_delta::key":0.1255663335998456,"hidden-states-l1/_layer_16_moe::routed_experts":0.030957378179654764,"update-absmax/_layer_44_gated_delta::A_log":0.0018949508666992188,"hidden-states-rms/_layer_46_gated_delta::beta":2.4423438132628963,"hidden-states-absmax/_layer_07_gated_delta::query":6.375,"hidden-states-l1/_layer_42_gated_delta::out_norm":0.11286266003401996,"param-l1/_layer_07_gated_delta::A_log":1.181315541267395,"hidden-states-absmax/_layer_05_gated_delta::gate":12.5,"grad-rms/layer_36/gated_delta::A_log":0.00010828180779739373,"hidden-states-rms/_layer_24_attention::k_layernorm":4.929260625933251,"hidden-states-mean/_layer_21_attention::o_proj":0.00013743581258857996,"hidden-states-rms/_layer_45_residual_mlp":8.929374040081196,"hidden-states-rms/_layer_02_gated_delta::g":4.97988154280988,"update-l1/_layer_07_gated_delta::dt_bias":0.0005891364417038858,"hidden-states-rms/_layer_50_attention::o_proj":2.6927382242044073,"grad-rms/layer_38/gated_delta::dt_bias":5.492998022132541e-06,"update-rms/_layer_18_gated_delta::conv1d":0.0008032661964667335,"param-rms/_layer_02_gated_delta::out_norm":0.15221228796184552,"param-mean/_layer_03_gated_delta::dt_bias":0.3764306902885437,"hidden-states-rms/_layer_01_gated_delta::core_attention":0.09561741245946132,"hidden-states-rms/_layer_39_moe::router":0.07216878508256834,"update-l1/_layer_14_gated_delta::out_norm":0.000732421875,"hidden-states-absmax/_layer_06_mlp":13.875,"hidden-states-mean/_layer_13_moe::routed_experts":3.1714014590905e-05,"param-mean/_layer_23_gated_delta::conv1d":0.0008821878000162542,"param-l1/attention::linear_qkv":0.028070020090256418,"hidden-states-absmax/_layer_41_gated_delta::query":28.5,"hidden-states-mean/_layer_29_gated_delta::out_norm":-0.0007136086619849399,"hidden-states-absmax/_layer_25_residual_mlp":500,"grad-absmax/layer_22/moe::expert_fc1":2.674219103937503e-06,"hidden-states-absmax/_layer_01_gated_delta::beta":10.5625,"hidden-states-absmax/_layer_18_gated_delta::g":13.8125,"update-rms/_layer_21_attention::o_proj":0.00016597320846953155,"hidden-states-rms/_layer_23_gated_delta::value":0.06182574740389129,"hidden-states-mean/_layer_56_moe::router":0.010353024211013192,"hidden-states-mean/_layer_17_gated_delta::beta":-2.1656346518557847,"update-mean/_layer_36_gated_delta::out_norm":0,"hidden-states-absmax/_layer_45_gated_delta::out_proj":166,"hidden-states-mean/_layer_51_gated_delta::value":0.025354688284359966,"hidden-states-l1/_layer_22_moe::routed_experts":0.05944597009478237,"hidden-states-mean/_layer_01_input_layernorm":0.002944738722337928,"hidden-states-l1/_layer_47_input_layernorm":0.3557345099132669,"hidden-states-mean/_layer_19_moe::routed_experts":-2.200478643654923e-05,"moe/layer_51/act_rms_min_to_median":0.2748028417531238,"hidden-states-rms/_layer_30_input_layernorm":0.3307555177577257,"hidden-states-rms/_layer_01_gated_delta::out_proj":0.30167263589439497,"hidden-states-l1/_layer_43_gated_delta::out_norm":0.10019205214882519,"grad-rms/layer_52/moe::shared_expert_fc1":8.285055398387902e-07,"hidden-states-mean/_layer_42_moe::router":0.01011896790964888,"param-absmax/_layer_11_gated_delta::out_proj":1.3125,"hidden-states-mean/_layer_08_gated_delta::in_proj":-0.10589711280136228,"grad-absmax/layer_32/gated_delta::out_proj":1.9302879081806168e-05,"grad-rms/layer_51/gated_delta::conv1d":0.00016464351556580155,"grad-rms/layer_05/moe::shared_expert_fc1":2.1626056716191109e-07,"hidden-states-l1/_layer_51_gated_delta::out_norm":0.2956389243172317,"hidden-states-mean/_layer_27_gated_delta::query":0.013377230346468729,"hidden-states-l1/_layer_27_pre_mlp_layernorm":0.11803558385796355,"hidden-states-rms/_layer_20_moe::shared_experts":0.17059564121386422,"hidden-states-rms/_layer_49_attention::k_layernorm":4.847311070578917,"param-absmax/_layer_40_gated_delta::conv1d":0.59375,"hidden-states-rms/_layer_45_gated_delta::out_proj":1.0413036461478453,"update-absmax/_layer_48_gated_delta::out_proj":0.00390625,"param-mean/_layer_04_gated_delta::out_norm":0.18660736083984375,"hidden-states-l1/_layer_55_mlp":0.2204377764985295,"hidden-states-rms/_layer_40_pre_mlp_layernorm":0.2194673453791917,"hidden-states-l1/_layer_40_gated_delta::out_proj":0.3432301095998418,"param-l1/_layer_38_gated_delta::dt_bias":0.4478888213634491,"hidden-states-l1/_layer_04_mlp":0.038413067270883,"grad-rms/layer_27/moe::expert_fc1":2.004128896573808e-08,"grad-absmax/layer_31/gated_delta::A_log":3.667143801067141e-06,"grad-rms/layer_06/gated_delta::A_log":4.1911875472929556e-05,"update-l1/_layer_09_gated_delta::dt_bias":0.0006138100870884955,"update-l1/_layer_45_gated_delta::dt_bias":0.0004629294853657484,"grad-rms/layer_47/moe::shared_experts":2.810532174417508e-06,"update-rms/_layer_41_gated_delta::dt_bias":0.0006253053031210772,"hidden-states-rms/_layer_54_residual_mlp":27.455685477820857,"grad-absmax/layer_56/gated_delta::conv1d":0.008772437460720539,"grad-rms/layer_55/moe::expert_fc2":1.183924536419997e-08,"hidden-states-absmax/gated_delta::query":59,"grad-absmax/layer_44/gated_delta::dt_bias":5.7183762692147866e-05,"hidden-states-l1/_layer_51_residual_mlp":5.613877011999728,"update-mean/_layer_48_gated_delta::conv1d":7.970279511937406e-06,"hidden-states-mean/_layer_50_residual_mlp":-0.0976051879054532,"hidden-states-mean/_layer_03_gated_delta::value":0.0013040587463597948,"update-rms/_layer_54_gated_delta::out_proj":0.0001916906782375916,"grad-absmax/layer_43/moe::expert_fc2":3.506691882648738e-06,"grad-absmax/layer_33/moe::expert_fc2":2.103380666085286e-06,"hidden-states-absmax/_layer_42_moe::router":0.9999511241912842,"moe/layer_50/act_rms_max":1.1399755477905273,"hidden-states-l1/_layer_10_gated_delta::beta":1.6991352676314435,"hidden-states-rms/_layer_32_gated_delta::beta":1.591954080855646,"hidden-states-absmax/_layer_13_moe::shared_experts":16.5,"hidden-states-absmax/_layer_20_gated_delta::in_proj":15.6875,"grad-absmax/layer_19/moe::shared_experts":6.655079687334364e-06,"hidden-states-rms/_layer_39_moe::shared_experts":0.17848766971385052,"update-rms/_layer_33_gated_delta::dt_bias":0.0007717744191489096,"hidden-states-l1/_layer_23_moe::routed_experts":0.0586641181570053,"grad-rms/layer_45/moe::expert_fc2":1.9564388298736424e-08,"grad-rms/layer_51/moe::expert_fc1":1.8634240415320765e-08,"param-absmax/_layer_48_gated_delta::out_proj":1.0234375,"grad-rms/layer_46/gated_delta::conv1d":0.00010081145566350867,"hidden-states-rms/_layer_32_moe::routed_experts":0.14322235191117166,"hidden-states-l1/_layer_12_residual_mlp":0.4112308515585484,"hidden-states-l1/_layer_11_gated_delta::value":0.03521041991834878,"hidden-states-rms/_layer_04_mlp":0.0579593442311846,"update-rms/_layer_54_gated_delta::A_log":0.0005387086392366538,"hidden-states-absmax/_layer_47_moe::router":0.9979207515716553,"hidden-states-mean/_layer_05_residual_mlp":0.0019054403471726593,"grad-absmax/layer_37/gated_delta::out_proj":5.676396540366113e-05,"hidden-states-l1/_layer_16_gated_delta::in_proj":0.7093872926864899,"moe/layer_26/act_rms_min":0.03264341503381729,"hidden-states-l1/_layer_53_gated_delta::gate":1.4130636946038835,"hidden-states-l1/_layer_22_attention::o_proj":0.2748147576826522,"hidden-states-rms/_layer_45_gated_delta::out_norm":0.5346986566694643,"hidden-states-rms/_layer_06_moe::shared_experts":0.043666679410078714,"hidden-states-l1/_layer_32_moe::router":0.010227828360053438,"hidden-states-mean/_layer_34_moe::shared_experts":-0.0015093461314181842,"grad-rms/layer_46/gated_delta::out_proj":6.52220495062375e-07,"update-mean/_layer_40_gated_delta::out_norm":0,"hidden-states-mean/_layer_37_gated_delta::key":0.021646107333434503,"update-mean/_layer_16_gated_delta::dt_bias":-0.00018156052101403475,"hidden-states-rms/_layer_47_residual_mlp":9.75603024249356,"param-rms/_layer_09_gated_delta::out_proj":0.04521972967653848,"hidden-states-l1/_layer_51_gated_delta::value":0.11081060598023008,"hidden-states-absmax/_layer_31_gated_delta::g":18.25,"moe/layer_35/act_rms_max":0.2695567011833191,"update-absmax/_layer_07_gated_delta::out_norm":0.001953125,"param-rms/_layer_29_gated_delta::dt_bias":0.23399893471789165,"update-l1/_layer_22_attention::o_proj":0.00012882678129244596,"hidden-states-absmax/_layer_04_pre_mlp_layernorm":2.5,"hidden-states-mean/gated_delta::value":0.004700446368861765,"param-mean/_layer_28_gated_delta::conv1d":4.147348226979375e-05,"param-l1/_layer_10_gated_delta::dt_bias":0.14829182624816895,"hidden-states-l1/_layer_03_mlp":0.035835292736922826,"grad-absmax/layer_10/moe::shared_experts":5.9251387938275e-06,"update-mean/_layer_55_gated_delta::conv1d":3.991950052295579e-06,"update-absmax/_layer_31_gated_delta::dt_bias":0.0016410648822784424,"hidden-states-absmax/_layer_30_residual_mlp":502,"hidden-states-l1/_layer_35_moe::routed_experts":0.08092801369785024,"update-rms/_layer_43_gated_delta::out_proj":0.0002044583630403523,"hidden-states-absmax/_layer_39_moe::routed_experts":26.625,"grad-absmax/layer_29/moe::shared_expert_fc2":2.2609028746956028e-05,"grad-rms/layer_28/gated_delta::conv1d":2.9217317944537206e-05,"grad-rms/moe::shared_expert_fc1":3.850923017566181e-07,"hidden-states-mean/_layer_12_input_layernorm":0.0018941932287595544,"hidden-states-absmax/_layer_31_moe::router":0.997978687286377,"update-rms/_layer_34_gated_delta::A_log":0.0006519877874334243,"hidden-states-l1/_layer_31_pre_mlp_layernorm":0.13597470652755578,"hidden-states-l1/_layer_28_gated_delta::g":7.805699325287262,"update-absmax/_layer_39_gated_delta::dt_bias":0.0018115341663360596,"hidden-states-absmax/_layer_43_residual_mlp":502,"grad-rms/layer_45/gated_delta::out_norm":1.1342925987899683e-05,"hidden-states-l1/_layer_47_attention::o_proj":0.8992012619020059,"param-absmax/_layer_30_gated_delta::conv1d":0.66015625,"hidden-states-absmax/_layer_28_input_layernorm":22,"hidden-states-absmax/_layer_23_input_layernorm":21,"hidden-states-l1/_layer_36_input_layernorm":0.24910536397759486,"grad-rms/layer_47/moe::shared_expert_fc2":2.111765394489072e-07,"hidden-states-absmax/_layer_26_pre_mlp_layernorm":3.390625,"param-rms/_layer_40_gated_delta::out_norm":0.7495961293953146,"hidden-states-l1/_layer_46_pre_mlp_layernorm":0.1832848111730589,"hidden-states-absmax/_layer_12_gated_delta::in_proj":23.875,"update-l1/_layer_40_gated_delta::conv1d":0.0006388357724063098,"param-l1/_layer_05_gated_delta::out_norm":0.2126312255859375,"hidden-states-absmax/_layer_07_gated_delta::value":4.96875,"hidden-states-rms/_layer_46_mlp":0.9015179649694788,"hidden-states-absmax/_layer_28_moe::shared_experts":11.8125,"grad-rms/layer_12/gated_delta::dt_bias":5.1647334344772654e-05,"update-rms/_layer_48_gated_delta::out_norm":0.0013810679320049757,"hidden-states-l1/_layer_25_gated_delta::out_norm":0.0766952054733283,"param-mean/_layer_05_gated_delta::out_proj":1.9017905287910253e-05,"hidden-states-mean/_layer_32_gated_delta::value":-0.005629987195728683,"grad-absmax/layer_25/gated_delta::dt_bias":3.5948196455137804e-06,"grad-rms/layer_24/moe::expert_fc1":1.8792082287183156e-08,"update-rms/_layer_42_gated_delta::out_norm":0.001760523083722651,"hidden-states-absmax/_layer_41_gated_delta::value":7.6875,"grad-absmax/layer_27/moe::router":5.7562436268199235e-05,"grad-rms/layer_42/moe::shared_expert_fc2":1.3766496980305373e-07,"grad-rms/layer_11/gated_delta::out_norm":3.0482217403136628e-05,"hidden-states-absmax/_layer_39_gated_delta::core_attention":0.236328125,"update-l1/_layer_37_gated_delta::conv1d":0.0006233980529941618,"hidden-states-absmax/_layer_13_residual_attn":488,"update-rms/_layer_23_gated_delta::out_norm":0.001600937139722168,"hidden-states-rms/_layer_15_pre_mlp_layernorm":0.11229941154289172,"update-absmax/_layer_38_gated_delta::out_norm":0.0078125,"hidden-states-mean/_layer_23_gated_delta::value":0.0108231542495673,"hidden-states-mean/_layer_20_mlp":-0.001685754131317631,"hidden-states-rms/_layer_11_pre_mlp_layernorm":0.09903834913254879,"hidden-states-absmax/gated_delta::beta":23.5,"hidden-states-rms/_layer_40_gated_delta::value":0.11480973563595166,"hidden-states-absmax/attention::q_layernorm":36,"hidden-states-absmax/_layer_42_gated_delta::beta":8.0625,"hidden-states-mean/_layer_42_residual_mlp":-0.09241652845341307,"moe/layer_25/act_rms_min":0.029794305562973022,"hidden-states-mean/_layer_19_moe::router":0.01026176203477574,"param-rms/_layer_54_gated_delta::out_norm":1.495918761694416,"param-rms/_layer_02_gated_delta::dt_bias":0.4937222635953125,"param-rms/_layer_24_attention::o_proj":0.04306486877413136,"update-absmax/_layer_36_gated_delta::out_norm":0.00390625,"grad-absmax/layer_25/gated_delta::out_norm":3.805302912951447e-05,"hidden-states-mean/_layer_06_mlp":-1.774976256137677e-05,"hidden-states-rms/_layer_01_gated_delta::out_norm":0.08780285631495131,"update-rms/_layer_26_gated_delta::dt_bias":0.0006736738976421366,"hidden-states-l1/_layer_06_gated_delta::beta":1.4290911470760281,"grad-absmax/layer_50/moe::expert_fc2":3.0134706321405247e-05,"grad-rms/layer_29/gated_delta::conv1d":1.273644737009507e-05,"grad-rms/layer_37/moe::router":3.102002666856825e-07,"grad-rms/layer_52/gated_delta::out_norm":6.032747761592504e-05,"update-mean/_layer_32_gated_delta::conv1d":-1.6087666153907776e-05,"hidden-states-rms/_layer_20_input_layernorm":0.3288415592640056,"hidden-states-rms/_layer_07_input_layernorm":0.31864616283493225,"hidden-states-l1/_layer_41_gated_delta::query":0.10961887802110648,"moe/layer_19/act_rms_min":0.01599031500518322,"grad-rms/layer_08/gated_delta::dt_bias":1.6546743603261254e-06,"hidden-states-l1/_layer_36_mlp":0.23883539305155194,"hidden-states-rms/_layer_52_residual_attn":24.434379254074717,"hidden-states-rms/_layer_38_gated_delta::g":2.535377208987893,"hidden-states-l1/_layer_19_input_layernorm":0.15894832121709837,"hidden-states-mean/_layer_17_residual_mlp":-0.001213645125130728,"update-mean/_layer_35_gated_delta::A_log":-0.00010191089677391574,"grad-absmax/layer_15/gated_delta::dt_bias":5.4989195632515475e-06,"hidden-states-mean/_layer_49_attention::o_proj":-0.002322797940408938,"hidden-states-l1/_layer_54_residual_attn":6.227116373433635,"param-rms/_layer_09_gated_delta::A_log":1.3319929906099344,"hidden-states-absmax/_layer_23_gated_delta::g":17.25,"hidden-states-rms/_layer_36_moe::routed_experts":0.14628289071718523,"update-l1/_layer_07_gated_delta::out_norm":0.0005950927734375,"grad-rms/layer_42/moe::router":4.560829169446634e-07,"param-l1/_layer_50_attention::o_proj":0.04412028566002846,"hidden-states-absmax/_layer_39_gated_delta::query":8.875,"param-l1/_layer_27_gated_delta::conv1d":0.03754806891083717,"update-rms/_layer_26_gated_delta::out_proj":0.00021014072090107247,"hidden-states-mean/_layer_04_pre_mlp_layernorm":5.9707428432349896e-05,"grad-rms/layer_33/gated_delta::out_proj":2.3723138125655918e-07,"hidden-states-mean/_layer_37_gated_delta::query":0.020839006330252447,"hidden-states-absmax/_layer_15_gated_delta::core_attention":0.0849609375,"grad-rms/layer_56/moe::shared_expert_fc2":2.7472582920097475e-07,"param-absmax/_layer_53_gated_delta::out_norm":2.15625,"hidden-states-mean/_layer_51_gated_delta::beta":-0.7426684192023777,"param-mean/gated_delta::A_log":1.0602322172145455,"hidden-states-mean/gated_delta::key":0.008409447061516648,"hidden-states-l1/_layer_31_gated_delta::in_proj":1.0121070025509495,"hidden-states-mean/_layer_28_moe::routed_experts":-0.00010840400665222046,"hidden-states-absmax/moe::routed_experts":378,"hidden-states-mean/_layer_07_input_layernorm":0.002139709213639404,"hidden-states-rms/_layer_56_gated_delta::in_proj":3.711196457990965,"hidden-states-rms/_layer_12_residual_attn":0.9502600090055027,"param-l1/gated_delta::A_log":1.1535850544365085,"hidden-states-mean/_layer_37_moe::shared_experts":-0.0006350124778985951,"hidden-states-absmax/_layer_51_gated_delta::value":12.5,"hidden-states-mean/_layer_52_mlp":0.0012831231944633358,"hidden-states-absmax/_layer_53_input_layernorm":25.75,"update-absmax/_layer_15_gated_delta::out_norm":0.0078125,"param-rms/_layer_19_gated_delta::out_proj":0.042088937749648406,"hidden-states-rms/_layer_36_gated_delta::out_norm":0.29709913441623886,"hidden-states-mean/_layer_48_moe::shared_experts":0.00018452853708961304,"hidden-states-absmax/_layer_45_moe::routed_experts":39.75,"grad-absmax/layer_34/moe::shared_experts":1.684849848970771e-05,"grad-rms/layer_53/gated_delta::out_norm":1.3983342478669517e-05,"moe/layer_55/act_rms_min":0.025337856262922287,"hidden-states-mean/_layer_14_moe::shared_experts":0.00022875898683550797,"hidden-states-rms/_layer_38_residual_mlp":7.3834786776826595,"param-rms/_layer_18_gated_delta::out_norm":0.8879196840256169,"param-l1/_layer_51_gated_delta::dt_bias":0.45040082931518555,"hidden-states-rms/_layer_56_residual_mlp":31.12616272718965,"hidden-states-absmax/_layer_36_input_layernorm":28.75,"grad-absmax/layer_14/moe::expert_fc1":3.319304141768953e-06,"hidden-states-absmax/_layer_45_gated_delta::query":38,"hidden-states-l1/_layer_26_mlp":0.1680970243331223,"hidden-states-mean/_layer_22_attention::core_attention":0.0010606295063724526,"hidden-states-l1/_layer_36_gated_delta::beta":1.8921763768406363,"hidden-states-rms/_layer_21_moe::router":0.07216878508256834,"grad-absmax/layer_16/gated_delta::out_norm":3.3492266084067523e-05,"grad-rms/layer_14/gated_delta::out_proj":4.20955168799118e-07,"hidden-states-l1/_layer_46_gated_delta::core_attention":0.0032691335477392134,"update-l1/_layer_43_gated_delta::dt_bias":0.0004933294840157032,"hidden-states-mean/_layer_12_moe::routed_experts":-1.4233331815817248e-05,"hidden-states-mean/_layer_47_attention::o_proj":-0.0021195910138149276,"param-rms/_layer_38_gated_delta::dt_bias":0.4816196273100096,"hidden-states-absmax/_layer_08_input_layernorm":9.6875,"hidden-states-mean/_layer_29_moe::router":0.010247696578136536,"hidden-states-mean/_layer_24_input_layernorm":0.005407584544430321,"update-rms/_layer_49_attention::o_proj":0.00026836308048844705,"hidden-states-l1/_layer_23_gated_delta::gate":0.8480414567043134,"grad-rms/layer_56/moe::expert_fc2":1.071537851172562e-08,"moe/layer_17/act_rms_median":0.054594654589891434,"update-rms/_layer_47_attention::o_proj":0.00025773664565488844,"param-absmax/_layer_26_gated_delta::dt_bias":0.48214736580848694,"hidden-states-l1/_layer_41_moe::shared_experts":0.1475985375164196,"grad-rms/layer_48/gated_delta::out_norm":1.6576101878131265e-05,"hidden-states-mean/_layer_22_residual_attn":-0.004938499193757676,"update-l1/_layer_45_gated_delta::out_proj":0.0001427552051609382,"hidden-states-rms/_layer_25_gated_delta::core_attention":0.004631293540932189,"hidden-states-mean/_layer_39_moe::router":0.010072172668126848,"hidden-states-absmax/_layer_11_gated_delta::out_proj":16.875,"param-absmax/_layer_53_gated_delta::conv1d":0.6953125,"hidden-states-l1/_layer_03_gated_delta::query":0.04310659087974916,"hidden-states-absmax/_layer_29_gated_delta::query":15.25,"param-rms/_layer_55_gated_delta::out_norm":1.374460287808881,"param-absmax/_layer_48_gated_delta::A_log":2.1937954425811768,"param-l1/_layer_10_gated_delta::conv1d":0.034472886472940445,"hidden-states-mean/_layer_30_mlp":-0.0015321300095996271,"update-rms/_layer_03_gated_delta::A_log":0.0007090222209791737,"hidden-states-l1/_layer_38_moe::shared_experts":0.10898024397852946,"update-mean/_layer_54_gated_delta::A_log":9.186379611492157e-05,"hidden-states-absmax/_layer_29_residual_attn":502,"hidden-states-rms/_layer_18_input_layernorm":0.26638110111121943,"update-rms/_layer_39_gated_delta::conv1d":0.0007458131016121123,"hidden-states-rms/_layer_41_gated_delta::key":0.23418980875673243,"update-absmax/_layer_41_gated_delta::out_proj":0.001953125,"lm-token-buffer-mib":235.4542694091797,"update-absmax/_layer_30_gated_delta::dt_bias":0.0010416855802759528,"grad-rms/layer_41/moe::shared_expert_fc2":2.280142230016576e-07,"hidden-states-mean/_layer_19_residual_attn":-0.0014836635113776935,"hidden-states-mean/_layer_18_gated_delta::value":0.001962086885373221,"param-rms/_layer_54_gated_delta::A_log":1.5871267135165013,"update-mean/_layer_09_gated_delta::conv1d":5.334538855095161e-06,"param-mean/_layer_55_gated_delta::conv1d":0.0010585299460217357,"grad-absmax/layer_07/moe::expert_fc1":1.2744122841468197e-06,"hidden-states-mean/_layer_15_mlp":0.0001696890451434979,"hidden-states-rms/_layer_43_gated_delta::key":0.21097227467152443,"param-l1/_layer_29_gated_delta::conv1d":0.03978372737765312,"hidden-states-rms/_layer_29_gated_delta::beta":2.8193525957725796,"hidden-states-mean/_layer_20_gated_delta::out_proj":-0.0007043954359019322,"param-mean/_layer_28_gated_delta::out_proj":1.8980221284436993e-05,"hidden-states-rms/_layer_42_gated_delta::gate":1.3208241456430658,"param-absmax/attention::o_proj":1.9453125,"hidden-states-l1/_layer_56_residual_mlp":7.04767120452776,"hidden-states-rms/_layer_06_gated_delta::core_attention":0.004306608052384612,"grad-rms/layer_42/gated_delta::dt_bias":4.266904380916602e-05,"hidden-states-absmax/_layer_16_gated_delta::query":4.8125,"hidden-states-absmax/_layer_17_gated_delta::value":2.9375,"update-rms/_layer_30_gated_delta::out_norm":0.0013593174713940482,"hidden-states-mean/_layer_11_gated_delta::beta":-1.4792178248426706,"hidden-states-mean/_layer_08_gated_delta::core_attention":-4.474331114087843e-06,"hidden-states-mean/_layer_03_input_layernorm":0.0028199510285199913,"hidden-states-mean/_layer_11_gated_delta::query":0.012198721581545065,"hidden-states-l1/_layer_50_attention::o_proj":0.8954918259968551,"update-absmax/_layer_16_gated_delta::conv1d":0.0029296875,"param-mean/_layer_57_lm_head":0.00013984426914248616,"hidden-states-absmax/_layer_43_moe::shared_experts":33,"hidden-states-l1/_layer_52_moe::routed_experts":0.08747462076140919,"update-rms/_layer_19_gated_delta::out_proj":0.0001978510890062335,"hidden-states-absmax/_layer_31_moe::shared_experts":19.625,"hidden-states-absmax/_layer_40_gated_delta::gate":27.125,"hidden-states-absmax/_layer_32_residual_mlp":502,"hidden-states-absmax/_layer_32_gated_delta::out_proj":33.75,"grad-rms/layer_46/gated_delta::out_norm":1.218518368987466e-05,"param-l1/_layer_23_gated_delta::dt_bias":0.15606355667114258,"update-absmax/_layer_09_gated_delta::out_norm":0.00390625,"update-mean/_layer_23_gated_delta::out_norm":-0.000274658203125,"hidden-states-rms/attention::q_layernorm":1.6342278286619925,"hidden-states-rms/_layer_30_gated_delta::key":0.12232322781419085,"hidden-states-absmax/_layer_45_input_layernorm":32.5,"hidden-states-absmax/_layer_14_gated_delta::in_proj":22.375,"hidden-states-absmax/_layer_10_moe::routed_experts":13.6875,"param-mean/_layer_45_gated_delta::dt_bias":0.4519028961658478,"grad-absmax/layer_24/moe::shared_experts":1.952436286956072e-05,"hidden-states-mean/_layer_48_gated_delta::key":0.01049649374890253,"hidden-states-mean/_layer_03_moe::shared_experts":0.0001432792367085277,"hidden-states-mean/_layer_33_mlp":-0.0029492316558041164,"hidden-states-absmax/_layer_29_gated_delta::value":8.5,"hidden-states-rms/_layer_55_pre_mlp_layernorm":0.16918284653408303,"hidden-states-l1/_layer_40_moe::shared_experts":0.12105756206006552,"hidden-states-rms/_layer_02_gated_delta::key":0.06570802424150421,"param-rms/_layer_39_gated_delta::dt_bias":0.4386038478644262,"hidden-states-rms/_layer_49_attention::q_layernorm":1.5396008153418241,"hidden-states-rms/_layer_19_gated_delta::value":0.04207594941575983,"update-l1/_layer_26_gated_delta::out_norm":0.00103759765625,"param-absmax/_layer_34_gated_delta::A_log":1.797461986541748,"grad-absmax/layer_36/moe::shared_expert_fc2":7.88593570177909e-06,"grad-rms/layer_45/gated_delta::out_proj":5.602833302291973e-07,"grad-absmax/layer_01/gated_delta::dt_bias":0.00024462025612592697,"hidden-states-mean/_layer_56_residual_mlp":0.11334584514449181,"hidden-states-absmax/_layer_52_moe::routed_experts":109.5,"hidden-states-mean/_layer_09_moe::router":0.010238929726907368,"update-l1/_layer_37_gated_delta::dt_bias":0.0005020522512495518,"grad-absmax/layer_38/gated_delta::conv1d":0.0011535349767655134,"hidden-states-rms/_layer_16_gated_delta::key":0.10514645612676564,"hidden-states-l1/_layer_45_input_layernorm":0.39671661218602805,"moe/layer_07/act_rms_min":0.008293099701404572,"param-mean/_layer_16_gated_delta::dt_bias":0.0027668364346027374,"param-l1/_layer_04_gated_delta::dt_bias":0.38426852226257324,"param-rms/_layer_16_gated_delta::A_log":1.0249375859308267,"hidden-states-absmax/_layer_04_input_layernorm":12.4375,"hidden-states-l1/_layer_37_gated_delta::in_proj":0.9227182762916731,"hidden-states-l1/_layer_05_residual_mlp":0.2257369280739937,"grad-absmax/layer_34/gated_delta::out_norm":3.268154978286475e-05,"hidden-states-absmax/_layer_17_gated_delta::query":10.75,"hidden-states-absmax/_layer_16_gated_delta::value":2.4375,"hidden-states-l1/_layer_09_moe::shared_experts":0.03836920197238946,"hidden-states-l1/_layer_45_residual_mlp":3.235287098322761,"hidden-states-rms/_layer_48_gated_delta::key":0.15757428445127605,"hidden-states-mean/_layer_21_attention::core_attention":-9.878996656752725e-05,"hidden-states-l1/_layer_25_pre_mlp_layernorm":0.12320525922837963,"grad-absmax/layer_10/moe::expert_fc2":3.6597170947061386e-06,"hidden-states-l1/_layer_38_gated_delta::core_attention":0.0003209789666210373,"update-mean/_layer_37_gated_delta::out_proj":-4.655338869952175e-08,"hidden-states-rms/_layer_09_gated_delta::query":0.11839840561093005,"param-absmax/_layer_20_gated_delta::A_log":2.251877546310425,"hidden-states-mean/_layer_26_moe::router":0.010278499145865304,"hidden-states-l1/_layer_37_gated_delta::value":0.03873092397820043,"param-absmax/_layer_46_gated_delta::out_proj":0.9375,"grad-absmax/layer_28/moe::shared_experts":1.191796673083445e-05,"hidden-states-absmax/_layer_22_attention::q_layernorm":25.5,"hidden-states-rms/_layer_35_residual_mlp":6.375222675730382,"grad-rms/layer_09/moe::router":2.8673885454857915e-07,"moe/layer_31/act_rms_median":0.11571318656206131,"hidden-states-rms/_layer_02_gated_delta::value":0.07426656391263182,"grad-rms/layer_52/gated_delta::A_log":1.355389525298126e-05,"update-rms/_layer_53_gated_delta::A_log":0.0006694332216257306,"grad-rms/layer_11/gated_delta::out_proj":5.024379318892298e-07,"param-rms/_layer_25_gated_delta::dt_bias":0.23069152733122533,"hidden-states-absmax/_layer_53_residual_mlp":2768,"hidden-states-rms/_layer_47_attention::core_attention":0.7653846430246306,"hidden-states-rms/_layer_08_gated_delta::core_attention":0.0004843176791056432,"moe/layer_05/act_rms_median":0.024000048637390137,"param-l1/_layer_17_gated_delta::conv1d":0.04142197594046593,"hidden-states-rms/_layer_24_attention::q_layernorm":1.6031087182970323,"hidden-states-rms/_layer_02_gated_delta::out_proj":0.09916811921596866,"grad-rms/layer_11/moe::shared_expert_fc1":2.9665718406529937e-07,"hidden-states-mean/_layer_12_gated_delta::core_attention":0.000497167057951338,"hidden-states-l1/_layer_15_gated_delta::out_norm":0.03325036191944622,"hidden-states-l1/_layer_21_attention::q_layernorm":1.2158931751123876,"hidden-states-l1/_layer_51_gated_delta::core_attention":0.004467660162467976,"update-l1/_layer_31_gated_delta::out_proj":0.00015647905820515007,"grad-rms/layer_33/gated_delta::A_log":1.1008089035440591e-06,"grad-rms/layer_25/moe::shared_experts":1.3224478860053133e-06,"hidden-states-l1/_layer_29_gated_delta::value":0.04671283995662212,"hidden-states-absmax/_layer_06_gated_delta::gate":35.5,"hidden-states-l1/_layer_09_gated_delta::g":5.184734008876563,"hidden-states-mean/_layer_33_gated_delta::beta":-0.8822147705810989,"grad-rms/layer_52/gated_delta::out_proj":8.210373611335579e-07,"param-absmax/_layer_04_gated_delta::out_norm":0.458984375,"hidden-states-l1/_layer_01_gated_delta::beta":2.3145630734638063,"grad-absmax/layer_13/moe::shared_expert_fc2":8.408228495682124e-06,"update-l1/_layer_03_gated_delta::conv1d":0.0006559166358783841,"param-l1/_layer_53_gated_delta::out_proj":0.03339335694909096,"hidden-states-l1/_layer_52_mlp":0.503436267785581,"update-absmax/_layer_01_gated_delta::out_norm":0.001953125,"update-l1/_layer_44_gated_delta::out_proj":0.00014034484047442675,"hidden-states-mean/_layer_20_gated_delta::value":0.00782862771494719,"param-absmax/_layer_01_gated_delta::out_norm":0.859375,"hidden-states-rms/_layer_26_residual_mlp":3.2306434083573325,"hidden-states-absmax/_layer_04_gated_delta::query":2.546875,"update-absmax/_layer_46_gated_delta::out_norm":0.0078125,"grad-absmax/layer_35/moe::router":4.774340050062165e-05,"param-absmax/_layer_53_gated_delta::A_log":2.4165871143341064,"hidden-states-absmax/gated_delta::g":70.5,"update-l1/_layer_06_gated_delta::conv1d":0.0006138051394373178,"update-rms/_layer_51_gated_delta::dt_bias":0.0006596532828882939,"hidden-states-absmax/_layer_44_gated_delta::out_norm":107.5,"update-rms/_layer_39_gated_delta::out_proj":0.00020490638718228112,"update-absmax/_layer_46_gated_delta::conv1d":0.00390625,"hidden-states-absmax/_layer_05_gated_delta::g":19.25,"hidden-states-mean/_layer_20_gated_delta::beta":-0.6846887093411338,"hidden-states-mean/_layer_55_input_layernorm":-0.0013590194740840166,"hidden-states-rms/gated_delta::in_proj":1.8002052358407103,"grad-rms/layer_33/gated_delta::out_norm":1.1346241121044817e-05,"hidden-states-absmax/_layer_28_gated_delta::query":2.5625,"param-rms/_layer_53_gated_delta::A_log":1.5107892157959792,"hidden-states-absmax/_layer_03_moe::shared_experts":4.40625,"hidden-states-rms/_layer_43_gated_delta::gate":1.557867982653167,"grad-absmax/layer_21/moe::router":7.292955706361681e-05,"grad-absmax/layer_16/moe::shared_expert_fc2":3.47329205396818e-06,"hidden-states-absmax/_layer_39_residual_mlp":502,"hidden-states-l1/_layer_07_gated_delta::out_proj":0.05947738104789031,"hidden-states-absmax/_layer_36_mlp":36,"hidden-states-absmax/_layer_55_gated_delta::core_attention":0.4609375,"hidden-states-absmax/_layer_19_gated_delta::out_norm":13.375,"param-rms/_layer_38_gated_delta::out_norm":1.0521295682108396,"hidden-states-absmax/_layer_53_moe::router":0.9996387958526611,"hidden-states-l1/_layer_17_gated_delta::query":0.07011766077490578,"hidden-states-mean/_layer_22_attention::o_proj":0.00017984376570210047,"moe/layer_15/act_rms_min_to_median":0.3333532462364064,"grad-rms/layer_54/moe::router":7.051943745026243e-07,"hidden-states-absmax/_layer_34_gated_delta::value":9.0625,"grad-rms/layer_20/moe::shared_expert_fc1":6.638719917930536e-07,"grad-absmax/layer_23/gated_delta::conv1d":0.0004864170332439244,"hidden-states-rms/_layer_43_gated_delta::query":0.5922330351400871,"grad-absmax/layer_16/moe::shared_experts":4.515309228736442e-06,"grad-absmax/layer_20/moe::router":0.00010576572822174057,"update-rms/_layer_24_attention::linear_qkv":0.00017642652663304702,"param-absmax/_layer_03_gated_delta::out_norm":0.34765625,"moe/layer_46/act_rms_min":0.08548688888549805,"hidden-states-l1/_layer_16_pre_mlp_layernorm":0.08556517488541304,"grad-absmax/layer_18/moe::shared_expert_fc2":4.038535735162441e-06,"hidden-states-rms/_layer_41_gated_delta::core_attention":0.0008553671672538917,"hidden-states-absmax/gated_delta::in_proj":113,"hidden-states-rms/_layer_21_moe::routed_experts":0.06784064860682891,"grad-rms/layer_08/moe::expert_fc1":1.5071346533865114e-08,"hidden-states-mean/_layer_37_gated_delta::in_proj":-0.27724327532387016,"grad-rms/layer_14/gated_delta::out_norm":2.0953094562103964e-05,"param-mean/_layer_52_gated_delta::A_log":1.2261991500854492,"hidden-states-rms/_layer_26_gated_delta::value":0.052046605283798314,"hidden-states-absmax/_layer_40_gated_delta::core_attention":0.474609375,"grad-absmax/layer_23/gated_delta::out_proj":6.015443432261236e-05,"hidden-states-rms/_layer_01_gated_delta::query":0.19570740093239206,"hidden-states-mean/_layer_32_moe::routed_experts":-0.00021955731962304404,"grad-rms/layer_49/moe::expert_fc1":1.9824670353196207e-08,"grad-rms/layer_53/gated_delta::out_proj":7.546197768515915e-07,"grad-absmax/layer_26/gated_delta::out_norm":2.232303268101532e-05,"hidden-states-mean/_layer_26_gated_delta::beta":-0.7620582296917713,"hidden-states-rms/_layer_53_gated_delta::g":6.228557643179787,"grad-absmax/layer_49/moe::shared_experts":1.746805355651304e-05,"update-rms/_layer_17_gated_delta::A_log":0.0006099004260994035,"hidden-states-absmax/_layer_40_gated_delta::g":20.375,"update-l1/_layer_31_gated_delta::dt_bias":0.0006328427698463202,"hidden-states-rms/_layer_15_input_layernorm":0.2775078305889298,"hidden-states-rms/_layer_38_moe::shared_experts":0.17260928824366065,"update-absmax/_layer_53_gated_delta::conv1d":0.00390625,"param-mean/_layer_42_gated_delta::A_log":1.1612833738327026,"hidden-states-mean/_layer_46_gated_delta::g":-4.405324507394374,"hidden-states-mean/_layer_31_residual_mlp":-0.03498919352760238,"grad-rms/layer_44/moe::shared_experts":3.3754923025931682e-06,"hidden-states-mean/_layer_37_gated_delta::gate":-0.6420300772225949,"hidden-states-l1/_layer_31_gated_delta::out_proj":0.23479392943309896,"moe/layer_44/act_rms_min":0.06822750717401505,"grad-absmax/layer_30/moe::expert_fc1":3.0016281016287394e-06,"hidden-states-rms/_layer_54_residual_attn":26.18122773509124,"hidden-states-absmax/_layer_06_moe::routed_experts":6.9375,"grad-absmax/layer_47/moe::router":6.59904326312244e-05,"hidden-states-absmax/_layer_22_attention::core_attention":7.53125,"hidden-states-mean/_layer_43_gated_delta::value":0.0013982933517373624,"hidden-states-mean/_layer_31_gated_delta::out_norm":-0.0015650797405561496,"grad-absmax/layer_40/moe::shared_expert_fc2":5.051635071140481e-06,"param-rms/gated_delta::out_norm":0.8693008239786563,"param-mean/_layer_53_gated_delta::out_proj":9.959782801161055e-06,"grad-absmax/layer_54/gated_delta::dt_bias":1.9994167814729735e-05,"update-l1/_layer_36_gated_delta::conv1d":0.0005190127994865179,"grad-absmax/layer_32/gated_delta::dt_bias":2.1182340788072906e-06,"hidden-states-mean/_layer_13_pre_mlp_layernorm":-0.000476786603479353,"update-rms/_layer_48_gated_delta::A_log":0.000661154279684335,"param-absmax/_layer_38_gated_delta::A_log":1.857330322265625,"hidden-states-mean/_layer_41_moe::routed_experts":-0.00030597583796002575,"hidden-states-absmax/_layer_52_gated_delta::beta":10.875,"hidden-states-mean/_layer_44_input_layernorm":0.0037022861408932376,"update-rms/_layer_17_gated_delta::out_norm":0.0017477808915132267,"grad-rms/layer_48/gated_delta::out_proj":5.897426582141159e-07,"param-rms/_layer_32_gated_delta::conv1d":0.09963110144536914,"param-l1/_layer_18_gated_delta::out_norm":0.8354949951171875,"hidden-states-mean/_layer_55_gated_delta::query":-0.006064671349039238,"grad-rms/layer_52/moe::router":1.4192039683433258e-06,"hidden-states-absmax/_layer_47_moe::routed_experts":27.25,"hidden-states-l1/_layer_55_gated_delta::g":5.23248114973082,"update-absmax/_layer_23_gated_delta::conv1d":0.0029296875,"grad-absmax/layer_42/moe::shared_experts":9.89453292277176e-06,"hidden-states-absmax/_layer_52_residual_mlp":2768,"update-mean/gated_delta::out_norm":-6.632902184311225e-05,"hidden-states-mean/_layer_24_mlp":-0.004286012191893123,"update-absmax/_layer_48_gated_delta::A_log":0.0014774799346923828,"grad-absmax/layer_04/gated_delta::out_proj":5.770120696979575e-05,"update-mean/attention::o_proj":-2.6939494952265316e-08,"hidden-states-rms/_layer_21_attention::q_layernorm":1.7144571599128735,"grad-rms/layer_14/moe::expert_fc1":2.1067829824598673e-08,"param-l1/_layer_18_gated_delta::dt_bias":0.11498408764600754,"hidden-states-l1/_layer_14_gated_delta::core_attention":0.0012465931098572147,"hidden-states-rms/_layer_35_gated_delta::core_attention":0.004950182773113324,"grad-rms/layer_42/gated_delta::A_log":4.3124272007433303e-05,"moe/layer_18/act_rms_max":0.13211356103420258,"hidden-states-l1/_layer_36_moe::router":0.010103617227652427,"hidden-states-absmax/_layer_37_gated_delta::core_attention":0.2080078125,"hidden-states-absmax/_layer_27_mlp":14.625,"update-l1/mlp::linear_fc1":0.00015347107546404004,"hidden-states-absmax/_layer_54_moe::shared_experts":29,"hidden-states-mean/_layer_10_moe::router":0.010248827824803642,"hidden-states-absmax/_layer_17_gated_delta::beta":17.625,"hidden-states-rms/_layer_19_gated_delta::core_attention":0.0013229730230246393,"grad-absmax/layer_10/moe::shared_expert_fc2":7.35859211999923e-05,"moe/layer_38/act_rms_max":0.30587559938430786,"hidden-states-mean/_layer_56_gated_delta::gate":-1.4833714346136373,"hidden-states-absmax/_layer_20_gated_delta::query":7.78125,"hidden-states-absmax/_layer_16_gated_delta::key":3.984375,"hidden-states-rms/_layer_31_moe::router":0.07216878508256834,"hidden-states-rms/_layer_36_gated_delta::in_proj":1.6084905654703179,"hidden-states-mean/_layer_23_residual_mlp":-0.004855901581752489,"hidden-states-rms/_layer_27_residual_mlp":3.403208986439289,"grad-absmax/layer_24/moe::router":8.835889457259327e-05,"moe/layer_26/act_rms_min_to_median":0.38352171733276547,"hidden-states-absmax/_layer_24_residual_attn":500,"hidden-states-rms/_layer_34_mlp":0.37855102457003054,"grad-absmax/layer_12/moe::shared_experts":1.713406527414918e-05,"hidden-states-absmax/_layer_42_gated_delta::g":15.5625,"hidden-states-absmax/_layer_29_gated_delta::core_attention":0.56640625,"param-mean/_layer_36_gated_delta::A_log":1.5009129047393799,"hidden-states-mean/_layer_13_moe::router":0.0102446517737153,"update-mean/_layer_53_gated_delta::conv1d":9.09231948753586e-06,"hidden-states-mean/_layer_23_gated_delta::core_attention":0.0004409654009561719,"hidden-states-rms/_layer_55_residual_attn":27.4721355125127,"hidden-states-mean/_layer_40_mlp":-0.0012848159898598182,"hidden-states-rms/_layer_15_residual_mlp":1.2557930023554775,"hidden-states-absmax/_layer_11_gated_delta::gate":22.625,"update-l1/_layer_30_gated_delta::out_proj":0.00014951203775126487,"grad-absmax/layer_45/moe::shared_expert_fc1":7.869202818255872e-05,"hidden-states-mean/_layer_48_pre_mlp_layernorm":0.0003657190681547306,"hidden-states-rms/_layer_22_attention::core_attention":0.3655803911838599,"param-mean/_layer_08_gated_delta::dt_bias":0.21712280809879303,"hidden-states-absmax/_layer_04_residual_mlp":43.5,"grad-absmax/layer_55/moe::shared_expert_fc2":2.732192569965264e-06,"moe/layer_24/act_rms_min":0.035357940942049026,"grad-rms/layer_07/moe::shared_experts":5.434495659457022e-07,"grad-absmax/layer_32/moe::shared_expert_fc2":3.782319481615559e-06,"hidden-states-l1/_layer_31_moe::router":0.01021219331418567,"update-rms/_layer_32_gated_delta::out_proj":0.0002138081585031287,"hidden-states-rms/_layer_47_pre_mlp_layernorm":0.27878316898325944,"hidden-states-l1/_layer_16_gated_delta::query":0.05405844274859938,"hidden-states-rms/embeddings":0.28919024013560396,"hidden-states-mean/_layer_40_gated_delta::gate":-0.5408440280966792,"hidden-states-rms/_layer_09_residual_attn":0.7373422408425856,"update-l1/_layer_34_gated_delta::A_log":0.000534401333425194,"hidden-states-l1/_layer_30_residual_attn":1.605091882359014,"update-rms/_layer_37_gated_delta::A_log":0.0006140211414069218,"hidden-states-l1/_layer_05_gated_delta::out_norm":0.014120812348522228,"hidden-states-mean/_layer_23_gated_delta::g":-5.919826736429541,"hidden-states-l1/_layer_25_residual_mlp":1.2148047941690439,"hidden-states-mean/_layer_50_attention::core_attention":-0.008562107659829576,"grad-absmax/layer_35/gated_delta::A_log":8.657684702484403e-06,"param-l1/_layer_15_gated_delta::conv1d":0.03833218291401863,"hidden-states-absmax/_layer_51_mlp":376,"grad-rms/layer_19/gated_delta::out_norm":1.5135342395565344e-05,"moe/layer_03/act_rms_median":0.016904203221201897,"update-rms/_layer_13_attention::linear_qkv":0.00019774393149054432,"grad-rms/layer_40/gated_delta::conv1d":1.3384848634864684e-05,"param-l1/_layer_46_gated_delta::out_norm":1.19207763671875,"hidden-states-rms/attention::o_proj":1.2367610250173096,"hidden-states-absmax/_layer_29_gated_delta::key":6.625,"hidden-states-absmax/_layer_14_moe::shared_experts":6.875,"hidden-states-l1/_layer_42_residual_attn":2.6880815849401456,"param-rms/_layer_12_gated_delta::A_log":1.1826581120989808,"hidden-states-rms/gated_delta::beta":2.308659444301513,"hidden-states-absmax/_layer_41_pre_mlp_layernorm":3.90625,"hidden-states-mean/_layer_50_moe::routed_experts":-0.00023981499354855395,"grad-absmax/layer_48/moe::shared_expert_fc1":0.00012269982835277915,"grad-absmax/layer_48/gated_delta::dt_bias":0.00014612337690778077,"update-l1/_layer_35_gated_delta::out_norm":0.0006103515625,"params-rms":0.060323475263954575,"param-mean/_layer_02_gated_delta::A_log":0.8620411157608032,"hidden-states-mean/_layer_19_gated_delta::beta":-2.0536534850184305,"update-l1/_layer_30_gated_delta::conv1d":0.0006389962509274483,"hidden-states-absmax/_layer_03_input_layernorm":12.1875,"grad-absmax/layer_35/moe::expert_fc1":3.5233315429650247e-06,"hidden-states-absmax/_layer_09_residual_attn":61.5,"update-mean/_layer_22_attention::linear_qkv":-1.3007109167517683e-08,"hidden-states-rms/_layer_37_moe::shared_experts":0.16000515881121236,"hidden-states-absmax/_layer_20_gated_delta::value":2.09375,"hidden-states-absmax/_layer_28_gated_delta::value":5.28125,"update-mean/_layer_51_gated_delta::A_log":-1.3833749108016491e-05,"hidden-states-absmax/_layer_56_pre_mlp_layernorm":7.65625,"grad-rms/layer_39/gated_delta::out_norm":6.6192703548478615e-06,"param-l1/_layer_04_gated_delta::out_norm":0.18660736083984375,"hidden-states-absmax/_layer_54_moe::router":0.9995597004890442,"update-l1/_layer_12_gated_delta::out_proj":0.00014180369907990098,"hidden-states-l1/_layer_27_mlp":0.1643274771365496,"update-l1/_layer_10_gated_delta::dt_bias":0.0004139735538046807,"hidden-states-absmax/_layer_18_gated_delta::core_attention":0.040283203125,"hidden-states-l1/_layer_23_gated_delta::out_norm":0.08846621524884883,"hidden-states-l1/_layer_26_gated_delta::query":0.0578444259279282,"hidden-states-absmax/_layer_39_gated_delta::beta":8.75,"hidden-states-absmax/_layer_31_mlp":30.25,"hidden-states-mean/_layer_46_gated_delta::core_attention":0.0011314128244489818,"hidden-states-mean/_layer_05_pre_mlp_layernorm":0.000335777625888154,"hidden-states-mean/_layer_29_mlp":-0.006304650853383643,"hidden-states-rms/_layer_35_gated_delta::value":0.0945559082521798,"hidden-states-mean/_layer_52_moe::shared_experts":0.0018561794306998767,"update-mean/_layer_49_attention::o_proj":-1.291324309704578e-07,"hidden-states-absmax/_layer_34_gated_delta::query":18.25,"grad-rms/layer_32/moe::expert_fc2":2.2465861979153086e-08,"hidden-states-mean/_layer_33_moe::router":0.010240630943080734,"hidden-states-l1/_layer_44_gated_delta::value":0.06161930029026273,"param-rms/_layer_28_gated_delta::conv1d":0.04862709385214554,"update-absmax/_layer_35_gated_delta::out_norm":0.0078125,"hidden-states-rms/_layer_29_gated_delta::g":9.37132999951982,"param-mean/_layer_19_gated_delta::dt_bias":0.04344157129526138,"update-absmax/_layer_14_gated_delta::out_norm":0.00390625,"param-mean/_layer_01_gated_delta::A_log":1.235562801361084,"param-l1/_layer_56_gated_delta::out_proj":0.02902667410671711,"grad-rms/layer_17/moe::expert_fc1":2.0195786856326347e-08,"param-mean/_layer_12_gated_delta::out_proj":-2.3273396436707117e-06,"hidden-states-mean/_layer_30_moe::shared_experts":-0.0008577440607279909,"grad-rms/layer_28/gated_delta::A_log":5.134259636846469e-06,"grad-absmax/layer_27/gated_delta::A_log":7.3839823926391546e-06,"update-mean/_layer_03_gated_delta::dt_bias":-5.124614108353853e-05,"update-absmax/_layer_31_gated_delta::conv1d":0.00390625,"hidden-states-absmax/_layer_43_gated_delta::out_proj":134,"hidden-states-l1/input_layernorm":0.2490034549240416,"hidden-states-rms/_layer_39_gated_delta::beta":1.4635691941244506,"param-l1/_layer_43_gated_delta::out_norm":0.8406982421875,"hidden-states-l1/_layer_03_pre_mlp_layernorm":0.03244132935301415,"hidden-states-l1/_layer_30_gated_delta::key":0.0583415436257772,"param-absmax/_layer_42_gated_delta::out_proj":0.6015625,"param-absmax/_layer_43_gated_delta::conv1d":0.515625,"hidden-states-mean/_layer_36_moe::shared_experts":-0.002420700909429897,"update-mean/_layer_43_gated_delta::out_proj":3.0258487981882354e-07,"hidden-states-absmax/mlp":564,"hidden-states-rms/_layer_11_gated_delta::query":0.13177362390202418,"update-mean/_layer_27_gated_delta::dt_bias":-2.060632687062025e-05,"_timestamp":1.7922371311085815e+09,"hidden-states-l1/_layer_03_moe::router":0.010353417229202149,"param-mean/_layer_56_gated_delta::out_proj":-2.2200327293830924e-05,"hidden-states-rms/_layer_51_gated_delta::gate":2.612975390946327,"param-absmax/_layer_23_gated_delta::out_proj":0.88671875,"hidden-states-l1/_layer_30_moe::routed_experts":0.07985025596885058,"hidden-states-rms/_layer_10_residual_attn":0.7801703514526733,"update-mean/_layer_30_gated_delta::out_norm":-9.1552734375e-05,"hidden-states-absmax/_layer_35_moe::routed_experts":23.25,"hidden-states-absmax/_layer_44_gated_delta::core_attention":0.310546875,"hidden-states-rms/_layer_02_gated_delta::out_norm":0.041087940771048284,"hidden-states-absmax/_layer_50_moe::shared_experts":148,"update-absmax/_layer_09_gated_delta::dt_bias":0.0015807747840881348,"grad-absmax/layer_48/moe::expert_fc1":1.906547367980238e-06,"update-l1/_layer_32_gated_delta::out_proj":0.00015019644342828542,"grad-rms/layer_46/moe::router":1.1354977748475735e-06,"param-rms/_layer_18_gated_delta::conv1d":0.07296795368270596,"update-absmax/_layer_01_gated_delta::dt_bias":0.0016464535146951675,"hidden-states-absmax/_layer_51_gated_delta::out_proj":2688,"hidden-states-mean/_layer_01_gated_delta::query":0.04267767953010553,"hidden-states-mean/_layer_28_gated_delta::query":0.01091895941423886,"hidden-states-mean/_layer_45_moe::shared_experts":0.000740907534338687,"hidden-states-l1/_layer_41_gated_delta::g":4.65823276737127,"update-rms/_layer_36_gated_delta::dt_bias":0.000523212464468646,"hidden-states-absmax/_layer_25_gated_delta::core_attention":0.1669921875,"grad-absmax/layer_19/gated_delta::A_log":2.2892321794643067e-05,"hidden-states-mean/_layer_51_gated_delta::g":-8.191986106870903,"grad-rms/layer_37/moe::shared_expert_fc1":1.6305856785799744e-07,"hidden-states-mean/_layer_15_gated_delta::out_norm":-0.0005192119897893953,"grad-absmax/layer_12/gated_delta::dt_bias":0.000290365656837821,"hidden-states-l1/_layer_09_moe::router":0.010238929726907368,"hidden-states-rms/_layer_16_gated_delta::value":0.05080224327980521,"hidden-states-mean/moe::shared_experts":-0.0005233789624377648,"update-l1/_layer_25_gated_delta::A_log":0.0005189398652873933,"hidden-states-absmax/_layer_07_moe::shared_experts":4.28125,"hidden-states-l1/_layer_01_residual_mlp":0.20690726720181313,"hidden-states-mean/_layer_17_pre_mlp_layernorm":-0.00045035162022803667,"hidden-states-rms/_layer_28_gated_delta::core_attention":0.0025377970338216773,"hidden-states-mean/_layer_32_gated_delta::beta":-1.053738123624101,"hidden-states-mean/_layer_38_input_layernorm":0.001226758963451858,"hidden-states-absmax/_layer_05_input_layernorm":11.1875,"grad-absmax/attention::linear_qkv":0.001140108099207282,"update-absmax/_layer_14_gated_delta::dt_bias":0.0015108603984117508,"param-absmax/_layer_29_gated_delta::A_log":1.652660608291626,"moe/layer_27/act_rms_min":0.03651992231607437,"update-rms/_layer_07_gated_delta::out_proj":0.00021777975255281334,"param-mean/_layer_27_gated_delta::out_proj":1.825183971959632e-05,"hidden-states-absmax/_layer_04_gated_delta::value":7.96875,"hidden-states-l1/_layer_29_gated_delta::in_proj":1.122801521470955,"hidden-states-mean/_layer_21_attention::q_layernorm":-0.04125416770627113,"hidden-states-rms/_layer_42_gated_delta::query":0.13515452351509089,"hidden-states-rms/_layer_36_gated_delta::value":0.0965941185050531,"update-rms/_layer_15_gated_delta::dt_bias":0.0007462130177798165,"hidden-states-absmax/_layer_17_gated_delta::out_proj":12.125,"hidden-states-l1/_layer_18_moe::router":0.010264109588918621,"hidden-states-rms/_layer_48_residual_mlp":10.091831685444566,"grad-rms/layer_45/gated_delta::dt_bias":1.8934245947367116e-05,"hidden-states-absmax/_layer_17_pre_mlp_layernorm":3.296875,"hidden-states-mean/mlp":-0.001025026840640388,"moe/layer_29/act_rms_min_to_median":0.35012564437298666,"param-rms/_layer_42_gated_delta::out_norm":1.2241398886358792,"hidden-states-absmax/_layer_03_gated_delta::out_norm":9.5,"grad-absmax/layer_46/moe::router":0.00013077848416287452,"update-l1/_layer_27_gated_delta::dt_bias":0.00048779038479551673,"update-absmax/_layer_51_gated_delta::out_norm":0.0078125,"hidden-states-mean/_layer_15_gated_delta::out_proj":-6.71198438772373e-05,"hidden-states-rms/_layer_29_gated_delta::out_proj":0.3492427155415752,"hidden-states-absmax/_layer_33_input_layernorm":34.75,"hidden-states-l1/_layer_01_gated_delta::query":0.10484601317000546,"grad-rms/layer_46/moe::shared_expert_fc1":6.774887099213464e-07,"hidden-states-rms/_layer_33_gated_delta::out_proj":0.34643679338673417,"update-rms/_layer_51_gated_delta::out_proj":0.00020414645654368065,"moe/layer_55/act_rms_max":0.32452091574668884,"param-absmax/_layer_43_gated_delta::dt_bias":0.8529086112976074,"hidden-states-l1/_layer_16_moe::router":0.010256911706036202,"hidden-states-l1/_layer_02_gated_delta::g":3.831812361694856,"grad-absmax/layer_52/gated_delta::out_proj":0.00011735144653357565,"param-mean/_layer_09_gated_delta::A_log":1.1726845502853394,"grad-rms/layer_29/moe::shared_experts":1.786536944483034e-06,"update-mean/_layer_09_gated_delta::out_proj":-5.4071083610551796e-08,"grad-rms/layer_38/moe::router":3.4329271029746763e-07,"update-l1/_layer_30_gated_delta::out_norm":0.000518798828125,"moe/layer_26/act_rms_max":0.18860803544521332,"update-rms/_layer_15_gated_delta::out_norm":0.001591602149707678,"hidden-states-l1/_layer_53_gated_delta::beta":2.2173836490373193,"hidden-states-rms/_layer_14_gated_delta::gate":1.3070704328717853,"grad-rms/layer_46/moe::shared_experts":3.889982440917516e-06,"update-l1/_layer_35_gated_delta::out_proj":0.00014563232252839953,"grad-absmax/layer_31/gated_delta::out_norm":1.8178048776462674e-05,"hidden-states-rms/_layer_46_gated_delta::value":0.141132586223087,"hidden-states-absmax/_layer_01_gated_delta::query":4.875,"hidden-states-l1/attention::core_attention":0.298471205392879,"hidden-states-mean/_layer_51_gated_delta::gate":0.06556165999909162,"param-mean/_layer_28_gated_delta::A_log":1.2028132677078247,"hidden-states-l1/_layer_36_gated_delta::query":0.04339029412153889,"grad-absmax/layer_45/gated_delta::conv1d":0.0023637504782527685,"hidden-states-l1/_layer_28_gated_delta::value":0.026549877038583403,"grad-absmax/layer_07/gated_delta::A_log":4.322796485212166e-06,"param-mean/_layer_38_gated_delta::out_proj":4.9355876399204135e-06,"grad-absmax/layer_43/gated_delta::conv1d":0.0006391883362084627,"param-l1/_layer_52_gated_delta::dt_bias":0.5770559906959534,"hidden-states-rms/_layer_35_gated_delta::gate":1.5801879126807643,"update-mean/_layer_33_gated_delta::A_log":-0.00024033163208514452,"hidden-states-absmax/_layer_40_gated_delta::out_proj":61.75,"hidden-states-mean/_layer_28_mlp":-0.0012318407256860407,"hidden-states-rms/_layer_02_pre_mlp_layernorm":0.08058734870878453,"hidden-states-mean/_layer_54_gated_delta::value":0.0017711941401593852,"update-rms/_layer_02_gated_delta::out_norm":0.0006686066375795485,"update-l1/_layer_48_gated_delta::conv1d":0.0005381117225624621,"param-absmax/_layer_30_gated_delta::out_proj":0.921875,"hidden-states-absmax/_layer_41_gated_delta::out_norm":85,"hidden-states-rms/_layer_54_moe::shared_experts":0.2670350395607199,"hidden-states-rms/_layer_23_moe::router":0.07216878508256834,"hidden-states-l1/_layer_38_input_layernorm":0.22871893390642808,"hidden-states-l1/_layer_16_gated_delta::out_norm":0.037918061812885026,"grad-rms/layer_15/gated_delta::A_log":1.7710679307695523e-06,"hidden-states-l1/_layer_36_gated_delta::in_proj":1.013191412767143,"grad-rms/layer_23/gated_delta::out_norm":1.4784914565109564e-05,"hidden-states-mean/_layer_55_gated_delta::in_proj":-0.46478012129984925,"grad-rms/layer_33/moe::expert_fc1":2.0164742807456963e-08,"hidden-states-rms/_layer_32_gated_delta::g":7.957708385760332,"param-mean/_layer_26_gated_delta::conv1d":-0.001080197747796774,"hidden-states-l1/_layer_30_moe::shared_experts":0.21257572920033088,"param-mean/_layer_20_gated_delta::out_proj":-3.530067260726355e-05,"grad-absmax/layer_18/gated_delta::out_norm":3.196230682078749e-05,"hidden-states-mean/_layer_18_residual_attn":-0.001069164263878973,"hidden-states-mean/_layer_44_gated_delta::query":0.02340180333278219,"update-absmax/_layer_18_gated_delta::out_norm":0.0078125,"hidden-states-l1/_layer_03_gated_delta::out_proj":0.05112288011585237,"hidden-states-l1/_layer_53_residual_mlp":6.09687401019025,"hidden-states-rms/_layer_29_gated_delta::out_norm":0.1662139312266793,"update-absmax/_layer_47_attention::linear_qkv":0.001953125,"hidden-states-absmax/_layer_51_residual_mlp":2720,"hidden-states-rms/_layer_33_gated_delta::out_norm":0.16428102816408602,"grad-rms/layer_50/moe::expert_fc2":2.2216123168385718e-08,"moe/layer_24/act_rms_min_to_median":0.3777265540164363,"hidden-states-mean/_layer_41_gated_delta::gate":-0.35623369887276873,"param-l1/_layer_07_gated_delta::out_norm":0.22312164306640625,"grad-rms/layer_51/moe::shared_experts":2.5669126418956145e-06,"param-rms/_layer_36_gated_delta::conv1d":0.042227207974221276,"update-l1/_layer_05_gated_delta::conv1d":0.0006602617213502526,"param-l1/_layer_19_gated_delta::conv1d":0.03359401598572731,"grad-rms/layer_08/gated_delta::out_norm":1.169240149112569e-05,"hidden-states-mean/_layer_51_gated_delta::key":-0.009590837385158228,"moe/layer_40/act_rms_min":0.09240801632404327,"update-l1/_layer_52_gated_delta::dt_bias":0.0005572158843278885,"moe/layer_45/act_rms_min":0.061045315116643906,"hidden-states-mean/_layer_31_residual_attn":-0.030968258123786226,"hidden-states-mean/_layer_39_mlp":-0.0007740658815767921,"grad-rms/layer_31/gated_delta::A_log":1.3507671346726522e-06,"hidden-states-mean/_layer_17_residual_attn":-0.0007161600786899124,"update-absmax/_layer_37_gated_delta::dt_bias":0.0012735724449157715,"hidden-states-rms/_layer_50_attention::core_attention":0.5882366944746839,"param-mean/_layer_15_gated_delta::conv1d":-0.0005944464937783778,"update-absmax/_layer_15_gated_delta::conv1d":0.00390625,"hidden-states-l1/_layer_33_gated_delta::in_proj":1.1572623695338091,"update-mean/_layer_29_gated_delta::out_norm":4.57763671875e-05,"grad-rms/layer_26/moe::expert_fc1":2.000261968494823e-08,"hidden-states-absmax/_layer_01_gated_delta::value":19,"update-l1/_layer_14_gated_delta::dt_bias":0.000467942125396803,"grad-absmax/layer_40/gated_delta::dt_bias":8.117406650853809e-06,"hidden-states-l1/_layer_01_gated_delta::in_proj":2.2131489142098992,"update-l1/_layer_35_gated_delta::A_log":0.00043622334487736225,"hidden-states-l1/_layer_09_mlp":0.06339484463868673,"hidden-states-l1/_layer_13_pre_mlp_layernorm":0.08080688337169688,"param-rms/_layer_08_gated_delta::conv1d":0.05948915190162601,"update-rms/_layer_35_gated_delta::out_norm":0.0016558422810364424,"update-rms/_layer_31_gated_delta::out_proj":0.00022154210186205723,"grad-absmax/layer_19/moe::shared_expert_fc1":2.4432592908851802e-05,"hidden-states-absmax/_layer_26_gated_delta::out_norm":35,"param-rms/_layer_42_gated_delta::A_log":1.308681200436149,"hidden-states-absmax/_layer_10_pre_mlp_layernorm":2.828125,"update-absmax/_layer_35_gated_delta::A_log":0.001815199851989746,"hidden-states-l1/_layer_17_gated_delta::g":6.374523922744265,"hidden-states-l1/_layer_10_gated_delta::gate":0.7422687104068649,"update-absmax/_layer_40_gated_delta::out_norm":0.0078125,"hidden-states-l1/_layer_04_pre_mlp_layernorm":0.03730386567757912,"update-absmax/_layer_11_gated_delta::A_log":0.0017574131488800049,"grad-rms/moe::router":6.845295615087241e-07,"hidden-states-absmax/_layer_56_gated_delta::key":3.875,"update-mean/_layer_46_gated_delta::conv1d":-1.3314031548361527e-06,"update-absmax/_layer_18_gated_delta::A_log":0.001127481460571289,"hidden-states-mean/_layer_41_gated_delta::g":-4.65307291186725,"param-rms/_layer_38_gated_delta::out_proj":0.04454449030147376,"hidden-states-absmax/gated_delta::out_proj":2688,"param-l1/_layer_13_attention::o_proj":0.027201533317565918,"grad-rms/layer_11/moe::router":2.876962970997e-07,"moe/layer_13/act_rms_max":0.08179859817028046,"param-mean/_layer_56_gated_delta::A_log":1.7857264280319214,"moe/layer_10/act_rms_min":0.013899832963943481,"update-mean/_layer_15_gated_delta::conv1d":-1.7616284821997397e-05,"grad-absmax/layer_23/moe::expert_fc2":2.223709088866599e-06,"grad-rms/layer_49/moe::router":7.011620193164658e-07,"grad-absmax/layer_52/moe::expert_fc2":1.331406383542344e-05,"grad-absmax/layer_30/gated_delta::out_norm":2.32582042372087e-05,"param-mean/_layer_26_gated_delta::out_norm":0.938232421875,"hidden-states-l1/_layer_15_gated_delta::g":5.187029980067012,"grad-rms/layer_47/moe::router":6.65294365111628e-07,"update-l1/_layer_51_gated_delta::A_log":0.0004953239695169032,"hidden-states-mean/_layer_29_gated_delta::value":-0.0009356041506466748,"grad-rms/layer_31/gated_delta::out_proj":2.6595111379711596e-07,"hidden-states-rms/_layer_56_gated_delta::beta":3.4675287386287663,"grad-absmax/layer_42/gated_delta::A_log":0.00024189887335523963,"hidden-states-mean/_layer_04_moe::shared_experts":0.00016244824409333546,"hidden-states-mean/_layer_22_moe::router":0.010251051202583504,"param-l1/gated_delta::conv1d":0.03663634254159976,"param-rms/_layer_27_gated_delta::A_log":1.0298039672927761,"hidden-states-l1/_layer_03_gated_delta::g":3.2045835500459137,"hidden-states-absmax/_layer_37_residual_mlp":502,"hidden-states-rms/_layer_06_gated_delta::key":0.1008738143613845,"grad-rms/layer_30/moe::expert_fc2":2.167719891814153e-08,"hidden-states-rms/_layer_38_gated_delta::out_proj":0.42384597524792894,"hidden-states-mean/_layer_50_attention::q_layernorm":0.04489872767314049,"param-l1/_layer_39_gated_delta::out_proj":0.034285470843315125,"hidden-states-rms/_layer_23_moe::shared_experts":0.2619970052113049,"moe/layer_32/act_rms_min_to_median":0.46470923402627196,"param-absmax/_layer_19_gated_delta::dt_bias":0.4987579882144928,"hidden-states-mean/_layer_22_moe::routed_experts":-4.0771882738319696e-05,"param-absmax/_layer_14_gated_delta::conv1d":0.8515625,"grad-absmax/layer_25/moe::shared_expert_fc1":3.0012217393959872e-05,"param-rms/_layer_19_gated_delta::dt_bias":0.16422940461015173,"param-l1/_layer_30_gated_delta::out_proj":0.03496966138482094,"grad-rms/layer_16/gated_delta::out_norm":1.3486292316309729e-05,"update-l1/_layer_19_gated_delta::A_log":0.0003771437332034111,"hidden-states-mean/_layer_15_gated_delta::value":-0.00017399386597910415,"update-absmax/_layer_32_gated_delta::dt_bias":0.0017774105072021484,"hidden-states-l1/_layer_38_moe::routed_experts":0.07748366939845601,"param-absmax/_layer_55_gated_delta::out_proj":1.703125,"grad-rms/layer_23/moe::shared_expert_fc1":7.018517327290498e-07,"hidden-states-l1/gated_delta::key":0.0858409730222339,"hidden-states-rms/_layer_42_residual_attn":7.946478137567281,"hidden-states-rms/_layer_55_gated_delta::gate":2.4071776792690245,"hidden-states-l1/_layer_52_gated_delta::key":0.16052215224768546,"hidden-states-mean/_layer_43_input_layernorm":0.00541875808835126,"hidden-states-mean/_layer_54_gated_delta::beta":-0.16646446781926963,"moe/layer_52/act_rms_min":0.03723013773560524,"hidden-states-mean/_layer_50_moe::router":0.010297616096952136,"hidden-states-mean/pre_mlp_layernorm":-6.746217372851324e-05,"grad-absmax/layer_18/gated_delta::A_log":1.3437091183732264e-05,"grad-rms/layer_02/gated_delta::out_norm":4.22082709813358e-05,"grad-absmax/layer_07/gated_delta::out_proj":1.1838977115985472e-05,"hidden-states-mean/_layer_41_gated_delta::in_proj":-0.1742122720047065,"update-rms/_layer_01_gated_delta::A_log":0.0007193689096811133,"grad-rms/layer_22/moe::expert_fc2":2.142141910150961e-08,"grad-absmax/layer_16/moe::expert_fc2":9.59081717155641e-07,"hidden-states-rms/gated_delta::core_attention":0.014979481428192935,"update-rms/_layer_08_gated_delta::conv1d":0.0008637914991886438,"update-absmax/_layer_03_gated_delta::A_log":0.0012930631637573242,"update-mean/_layer_16_gated_delta::out_norm":0.000213623046875,"param-mean/_layer_52_gated_delta::conv1d":-0.00035937639768235385,"grad-absmax/layer_34/moe::shared_expert_fc2":2.9957811875647167e-06,"grad-rms/layer_52/gated_delta::conv1d":0.0003922912136509492,"hidden-states-absmax/_layer_05_gated_delta::key":4.625,"grad-rms/layer_36/moe::shared_expert_fc1":2.2095952393340652e-07,"param-mean/_layer_48_gated_delta::out_proj":-4.116573472856544e-05,"update-rms/_layer_05_gated_delta::out_norm":0.0009765625,"update-l1/_layer_31_gated_delta::out_norm":0.00067138671875,"param-mean/_layer_42_gated_delta::out_proj":-4.5315417082747445e-05,"grad-absmax/layer_51/moe::shared_expert_fc1":6.132820271886885e-05,"hidden-states-rms/_layer_20_residual_attn":1.4743758131085636,"hidden-states-mean/_layer_14_gated_delta::in_proj":-0.19026436655156326,"hidden-states-absmax/_layer_35_gated_delta::query":15.25,"param-mean/_layer_48_gated_delta::conv1d":-0.0007445709779858589,"hidden-states-mean/_layer_27_gated_delta::g":-7.334648529845301,"param-mean/_layer_56_gated_delta::conv1d":0.0003754292265512049,"hidden-states-l1/_layer_50_residual_attn":4.211162115473999,"hidden-states-rms/_layer_04_gated_delta::gate":0.9883614293836624,"param-mean/_layer_23_gated_delta::out_proj":5.006149149267003e-05,"hidden-states-absmax/_layer_32_moe::routed_experts":17.25,"hidden-states-mean/_layer_40_gated_delta::in_proj":-0.2862690147370717,"grad-absmax/layer_25/moe::expert_fc1":2.719583790167235e-06,"hidden-states-mean/_layer_15_residual_attn":-0.0005368136393019425,"grad-rms/layer_55/gated_delta::conv1d":0.0001391991455161734,"param-l1/_layer_56_gated_delta::A_log":1.7857264280319214,"moe/layer_05/act_rms_min_to_median":0.24456216883919363,"hidden-states-l1/_layer_18_gated_delta::out_norm":0.0478599687349759,"update-l1/_layer_44_gated_delta::out_norm":0.0006103515625,"hidden-states-absmax/_layer_28_residual_mlp":502,"hidden-states-absmax/gated_delta::key":23.625,"grad-rms/layer_34/gated_delta::out_norm":1.204734922470409e-05,"param-rms/_layer_51_gated_delta::conv1d":0.07925419151942582,"hidden-states-absmax/_layer_41_gated_delta::beta":11.625,"hidden-states-rms/_layer_31_gated_delta::key":0.2329618747372725,"hidden-states-mean/_layer_45_gated_delta::key":-0.006365105337215106,"moe/layer_36/act_rms_min":0.07062432914972305,"hidden-states-mean/_layer_35_residual_attn":-0.05273939180247686,"hidden-states-l1/_layer_06_gated_delta::value":0.037737903027284504,"update-absmax/_layer_51_gated_delta::dt_bias":0.0018371939659118652,"update-absmax/_layer_41_gated_delta::dt_bias":0.0014465153217315674,"grad-absmax/layer_09/moe::shared_experts":4.900934072793461e-06,"hidden-states-absmax/_layer_34_moe::routed_experts":23.375,"hidden-states-mean/_layer_31_gated_delta::out_proj":-0.0040127840020101075,"update-mean/_layer_14_gated_delta::out_proj":-1.9247694638124813e-07,"hidden-states-mean/_layer_45_input_layernorm":0.004722721506383598,"grad-rms/layer_47/attention::o_proj":3.786806698321469e-07,"param-l1/_layer_46_gated_delta::conv1d":0.03483089804649353,"param-mean/_layer_37_gated_delta::conv1d":9.469164069741964e-05,"moe/layer_51/act_rms_max":0.6250808835029602,"param-rms/_layer_02_gated_delta::A_log":1.2797232114026589,"moe/layer_25/act_rms_max":0.18322411179542542,"hidden-states-rms/_layer_56_moe::router":0.07216878508256834,"hidden-states-mean/_layer_40_gated_delta::g":-6.814038135294692,"hidden-states-absmax/_layer_50_attention::o_proj":696,"update-mean/_layer_15_gated_delta::out_norm":-0.000213623046875,"grad-rms/layer_51/moe::router":1.7037908258508378e-06,"hidden-states-absmax/attention::core_attention":20.75,"update-absmax/_layer_42_gated_delta::conv1d":0.0032958984375,"hidden-states-mean/_layer_31_gated_delta::core_attention":6.397275900329757e-05,"hidden-states-rms/_layer_24_moe::shared_experts":0.404703742888453,"grad-rms/layer_29/moe::router":6.681201795107162e-07,"hidden-states-rms/_layer_07_moe::router":0.07216878508256834,"hidden-states-rms/_layer_56_moe::routed_experts":0.17583618347640348,"param-absmax/_layer_06_gated_delta::A_log":2.0880770683288574,"hidden-states-absmax/_layer_27_residual_attn":502,"hidden-states-l1/_layer_16_gated_delta::key":0.05821970426675132,"hidden-states-rms/_layer_49_moe::routed_experts":0.3256650463178026,"param-mean/_layer_47_attention::o_proj":9.811509698920418e-06,"update-absmax/_layer_28_gated_delta::out_proj":0.001953125,"hidden-states-absmax/_layer_05_residual_mlp":46,"hidden-states-rms/_layer_32_input_layernorm":0.6503312754939117,"grad-absmax/layer_15/gated_delta::A_log":5.47035733688972e-06,"hidden-states-mean/_layer_22_attention::q_layernorm":-1.1598110650254565e-05,"hidden-states-l1/_layer_22_attention::q_layernorm":1.1872815116979494,"param-absmax/_layer_36_gated_delta::out_proj":1.390625,"grad-absmax/layer_29/moe::shared_experts":1.5416520909639075e-05,"hidden-states-rms/_layer_14_pre_mlp_layernorm":0.11121848874681868,"hidden-states-l1/_layer_34_gated_delta::gate":1.2119771494838179,"grad-rms/layer_03/moe::router":6.329462827760452e-07,"hidden-states-absmax/_layer_26_mlp":14.0625,"update-mean/_layer_08_gated_delta::conv1d":1.2829355910071172e-05,"update-absmax/_layer_13_attention::o_proj":0.00390625,"hidden-states-absmax/_layer_09_gated_delta::core_attention":0.240234375,"hidden-states-rms/_layer_26_gated_delta::in_proj":1.2749177788644064,"param-absmax/_layer_18_gated_delta::out_norm":1.890625,"hidden-states-mean/_layer_17_gated_delta::gate":-0.03679893977033374,"hidden-states-l1/_layer_11_residual_mlp":0.3594959508771838,"hidden-states-l1/_layer_32_residual_attn":1.79358134193198,"grad-rms/layer_22/attention::o_proj":1.1091255328475979e-06,"update-l1/_layer_38_gated_delta::A_log":0.0004364540218375623,"hidden-states-mean/_layer_19_gated_delta::value":0.0017035394842299205,"hidden-states-l1/_layer_19_gated_delta::out_norm":0.05258571983405588,"hidden-states-absmax/_layer_15_moe::shared_experts":4.84375,"grad-rms/layer_55/moe::shared_expert_fc1":1.3851482345598752e-07,"hidden-states-rms/_layer_55_mlp":0.5377189688272463,"moe/layer_36/act_rms_median":0.13177630305290222,"hidden-states-absmax/_layer_08_gated_delta::gate":12.8125,"param-rms/_layer_30_gated_delta::A_log":1.1910437861160086,"hidden-states-rms/_layer_23_gated_delta::g":6.492788662546131,"hidden-states-rms/_layer_39_pre_mlp_layernorm":0.2132605344835862,"update-l1/_layer_15_gated_delta::conv1d":0.0006525223143398762,"grad-absmax/layer_20/moe::expert_fc1":4.793871994479559e-06,"hidden-states-absmax/_layer_01_gated_delta::core_attention":5.75,"hidden-states-rms/_layer_35_moe::router":0.07216878508256834,"hidden-states-mean/_layer_43_moe::router":0.010158511251570582,"grad-rms/moe::shared_expert_fc2":3.1099149330307e-07,"grad-absmax/layer_27/moe::expert_fc1":2.753738272076589e-06,"hidden-states-l1/_layer_11_gated_delta::g":6.382309720791545,"hidden-states-mean/_layer_11_residual_mlp":-0.0003942272630157784,"grad-rms/layer_01/gated_delta::dt_bias":4.324812970744259e-05,"param-mean/_layer_43_gated_delta::out_norm":0.8406982421875,"hidden-states-mean/_layer_39_moe::routed_experts":-0.00015327262554098046,"update-l1/_layer_26_gated_delta::out_proj":0.00014718562306370586,"hidden-states-rms/_layer_05_input_layernorm":0.3193489752361241,"grad-absmax/layer_49/moe::expert_fc1":3.811136139120208e-06,"hidden-states-l1/_layer_34_gated_delta::g":5.688711367691918,"moe/layer_13/act_rms_min_to_median":0.3490790725411094,"hidden-states-rms/_layer_40_gated_delta::query":0.4941928964615022,"hidden-states-rms/_layer_07_mlp":0.07781909284147062,"update-rms/_layer_51_gated_delta::A_log":0.000654444016873982,"update-rms/_layer_38_gated_delta::conv1d":0.0007639130326033867,"grad-rms/attention::o_proj":9.648535232175898e-07,"hidden-states-mean/_layer_32_gated_delta::g":-7.532882634744959,"hidden-states-rms/_layer_21_attention::o_proj":0.3197994617035094,"hidden-states-mean/_layer_02_gated_delta::core_attention":0.00039797473087978133,"hidden-states-absmax/_layer_40_pre_mlp_layernorm":4.4375,"grad-rms/lm_head":1.0540706181194214e-06,"param-rms/_layer_55_gated_delta::out_proj":0.04157551450313166,"param-absmax/_layer_44_gated_delta::out_proj":1,"hidden-states-mean/_layer_29_input_layernorm":0.0010717668451239878,"hidden-states-l1/_layer_27_gated_delta::in_proj":0.7074557841223996,"hidden-states-mean/_layer_42_mlp":-0.0008346144872071727,"hidden-states-l1/_layer_23_gated_delta::beta":1.7303735888449656,"hidden-states-l1/_layer_38_moe::router":0.010064465662156052,"hidden-states-l1/_layer_15_input_layernorm":0.15188871644600827,"update-l1/_layer_25_gated_delta::dt_bias":0.000521449779625982,"hidden-states-l1/_layer_50_attention::core_attention":0.36437670759877056,"hidden-states-rms/_layer_14_gated_delta::core_attention":0.002595980122982873,"update-rms/_layer_04_gated_delta::out_proj":0.00021770237282290055,"update-l1/_layer_27_gated_delta::A_log":0.00048531172797083855,"update-rms/_layer_03_gated_delta::conv1d":0.0008385338360139785,"moe/layer_36/act_rms_min_to_median":0.5359410011825918,"param-rms/_layer_42_gated_delta::out_proj":0.044568086225021245,"hidden-states-absmax/_layer_22_input_layernorm":35,"grad-rms/layer_25/gated_delta::conv1d":1.1133705921813444e-05,"hidden-states-rms/_layer_15_gated_delta::g":6.121944730849396,"update-l1/_layer_45_gated_delta::out_norm":0.0003662109375,"update-mean/_layer_46_gated_delta::out_norm":-0.00067138671875,"hidden-states-absmax/_layer_47_attention::core_attention":17.25,"hidden-states-rms/_layer_08_moe::router":0.07216878508256834,"grad-absmax/layer_52/gated_delta::A_log":7.128215656848624e-05,"hidden-states-absmax/_layer_10_moe::router":0.9974734783172607,"hidden-states-mean/_layer_12_gated_delta::query":0.016801303392804074,"moe/layer_49/act_rms_max":0.9176084399223328,"hidden-states-absmax/_layer_02_gated_delta::in_proj":18.625,"hidden-states-rms/_layer_19_mlp":0.1903963417790848,"param-l1/gated_delta::out_proj":0.033913149745488654,"hidden-states-mean/_layer_30_gated_delta::g":-5.00415813972451,"grad-absmax/layer_05/gated_delta::out_proj":6.80048979120329e-05,"param-mean/_layer_54_gated_delta::out_norm":1.4852294921875,"param-l1/_layer_52_gated_delta::A_log":1.3062947988510132,"param-absmax/_layer_51_gated_delta::out_norm":2.078125,"hidden-states-rms/_layer_23_gated_delta::query":0.18664806371501874,"hidden-states-rms/_layer_18_gated_delta::beta":1.6133678718922764,"hidden-states-absmax/_layer_15_residual_attn":494,"hidden-states-mean/_layer_22_input_layernorm":0.007310966901298392,"param-absmax/_layer_19_gated_delta::out_norm":1.40625,"hidden-states-absmax/_layer_22_residual_attn":500,"hidden-states-mean/_layer_43_mlp":-0.0013553032637376504,"grad-absmax/layer_44/moe::shared_experts":3.0857492674840614e-05,"grad-absmax/layer_01/mlp::linear_fc1":2.8084206860512495e-05,"hidden-states-absmax/_layer_32_gated_delta::beta":8.3125,"param-l1/_layer_44_gated_delta::A_log":1.1538420915603638,"grad-rms/layer_35/moe::shared_expert_fc1":2.341612087122678e-07,"param-absmax/_layer_23_gated_delta::conv1d":0.8203125,"param-absmax/_layer_19_gated_delta::A_log":1.881946086883545,"hidden-states-mean/_layer_49_attention::k_layernorm":-0.06808055258757503,"hidden-states-rms/_layer_33_moe::router":0.07216878508256834,"param-l1/_layer_05_gated_delta::out_proj":0.03571120277047157,"grad-absmax/layer_11/moe::shared_expert_fc2":5.518908437807113e-06,"param-absmax/_layer_09_gated_delta::out_norm":0.8046875,"moe/layer_22/act_rms_max":0.15996278822422028,"grad-rms/layer_03/gated_delta::out_norm":4.472568080078057e-05,"param-absmax/_layer_39_gated_delta::conv1d":0.482421875,"hidden-states-l1/_layer_04_gated_delta::out_proj":0.04973647963839527,"grad-absmax/layer_48/gated_delta::A_log":0.00017381837824359536,"grad-absmax/layer_12/gated_delta::conv1d":0.004221734125167131,"hidden-states-l1/_layer_30_mlp":0.31320969023820044,"param-l1/_layer_15_gated_delta::dt_bias":0.20414945483207703,"hidden-states-l1/_layer_03_residual_attn":0.18524115505709113,"update-absmax/_layer_52_gated_delta::A_log":0.001692056655883789,"hidden-states-absmax/_layer_12_residual_mlp":484,"update-mean/_layer_31_gated_delta::dt_bias":-0.00010123965330421925,"hidden-states-absmax/_layer_35_gated_delta::value":7.125,"param-absmax/_layer_31_gated_delta::out_proj":0.83203125,"grad-absmax/layer_39/gated_delta::A_log":6.188676707097329e-06,"grad-rms/layer_41/moe::router":3.786721111791765e-07,"hidden-states-rms/_layer_33_moe::routed_experts":0.1502180762901769,"hidden-states-mean/_layer_05_gated_delta::query":0.013649333964208837,"update-l1/_layer_48_gated_delta::dt_bias":0.0005542468279600143,"hidden-states-mean/_layer_49_residual_attn":-0.10475767907591808,"update-l1/_layer_32_gated_delta::out_norm":0.0005340576171875,"update-mean/_layer_48_gated_delta::out_norm":-0.000244140625,"param-l1/_layer_44_gated_delta::conv1d":0.03641430661082268,"update-mean/_layer_50_attention::o_proj":7.024977577430036e-08,"hidden-states-l1/_layer_26_gated_delta::out_proj":0.20587855051500206,"hidden-states-rms/_layer_35_mlp":0.44326867964628014,"hidden-states-rms/_layer_40_moe::shared_experts":0.1853576206893267,"grad-absmax/layer_52/gated_delta::conv1d":0.026494406163692474,"update-absmax/_layer_08_gated_delta::out_proj":0.00390625,"update-absmax/_layer_11_gated_delta::out_norm":0.0029296875,"hidden-states-rms/_layer_04_moe::shared_experts":0.03933423948789289,"param-absmax/_layer_11_gated_delta::conv1d":0.515625,"grad-rms/layer_43/moe::expert_fc1":2.0436665193690755e-08,"moe/layer_06/act_rms_max":0.05206863582134247,"grad-rms/layer_39/gated_delta::A_log":2.0482043834133754e-06,"hidden-states-l1/_layer_06_gated_delta::g":5.899646508651555,"hidden-states-mean/_layer_23_residual_attn":-0.0041253357455534635,"update-l1/_layer_24_attention::o_proj":0.00013914423470851034,"hidden-states-absmax/_layer_46_mlp":218,"hidden-states-rms/_layer_03_moe::router":0.07216878508256834,"hidden-states-mean/_layer_03_mlp":0.00017123425499814064,"hidden-states-l1/_layer_36_gated_delta::gate":1.0279967818256357,"hidden-states-mean/_layer_21_moe::shared_experts":-0.0016252289833773952,"hidden-states-rms/_layer_44_gated_delta::gate":1.874727341046941,"hidden-states-mean/_layer_19_gated_delta::key":0.017889937655918318,"grad-rms/layer_28/moe::expert_fc1":1.9993651159773887e-08,"grad-absmax/layer_38/moe::expert_fc2":3.111620117124403e-06,"param-l1/_layer_42_gated_delta::A_log":1.1869438886642456,"hidden-states-mean/_layer_52_gated_delta::core_attention":0.003738387933479301,"hidden-states-absmax/_layer_18_pre_mlp_layernorm":3.3125,"hidden-states-rms/_layer_10_moe::routed_experts":0.03602866444932224,"update-mean/_layer_33_gated_delta::out_proj":1.2683882744113362e-07,"update-absmax/_layer_18_gated_delta::conv1d":0.00390625,"hidden-states-absmax/_layer_16_gated_delta::out_proj":10.375,"hidden-states-l1/_layer_52_moe::shared_experts":0.4041679089269529,"grad-absmax/layer_23/moe::shared_expert_fc2":2.33292012126185e-05,"update-absmax/_layer_16_gated_delta::out_proj":0.00390625,"hidden-states-rms/_layer_22_pre_mlp_layernorm":0.1532728796286563,"hidden-states-absmax/_layer_11_pre_mlp_layernorm":6.9375,"update-rms/_layer_43_gated_delta::dt_bias":0.000615775859225475,"update-l1/_layer_12_gated_delta::out_norm":0.000701904296875,"grad-absmax/layer_36/moe::expert_fc2":1.1476192412374076e-05,"update-rms/_layer_52_gated_delta::dt_bias":0.0007164795260788948,"hidden-states-l1/_layer_40_moe::routed_experts":0.08999912704196271,"grad-absmax/layer_51/gated_delta::out_norm":0.00022293049551080912,"hidden-states-l1/_layer_20_pre_mlp_layernorm":0.09211577083060393,"hidden-states-rms/_layer_41_gated_delta::g":5.154013715948752,"hidden-states-rms/_layer_26_moe::router":0.07216878508256834,"grad-absmax/layer_53/moe::shared_expert_fc2":3.492692485451698e-05,"param-rms/_layer_49_attention::o_proj":0.056251556404754294,"hidden-states-mean/_layer_20_gated_delta::out_norm":-0.001725385171220919,"hidden-states-absmax/_layer_03_residual_mlp":37,"hidden-states-l1/_layer_39_pre_mlp_layernorm":0.15206148052948482,"hidden-states-l1/_layer_56_moe::router":0.010353024211013192,"grad-absmax/layer_46/gated_delta::A_log":4.629552859114483e-05,"hidden-states-mean/_layer_43_pre_mlp_layernorm":0.00026301417026278916,"hidden-states-l1/_layer_26_pre_mlp_layernorm":0.11862406058598286,"param-mean/_layer_49_attention::o_proj":-2.113492701028008e-05,"hidden-states-mean/_layer_38_residual_attn":-0.0819552936500661,"hidden-states-l1/_layer_52_gated_delta::g":7.613327933946602,"hidden-states-mean/_layer_38_moe::router":0.010064465662156052,"hidden-states-mean/_layer_06_pre_mlp_layernorm":0.00015213499495133698,"hidden-states-l1/_layer_06_gated_delta::gate":0.960496182500861,"moe/layer_07/act_rms_min_to_median":0.27582445647119724,"hidden-states-l1/_layer_30_gated_delta::out_norm":0.07606721831169347,"hidden-states-mean/_layer_47_attention::q_layernorm":-0.007800963705893005,"param-l1/_layer_02_mlp::linear_fc1":0.03661181032657623,"update-l1/_layer_40_gated_delta::A_log":0.0005351456347852945,"hidden-states-rms/_layer_26_gated_delta::beta":1.4243911458402236,"hidden-states-absmax/_layer_32_gated_delta::core_attention":0.69921875,"hidden-states-rms/_layer_29_pre_mlp_layernorm":0.18719016862605584,"hidden-states-mean/_layer_38_pre_mlp_layernorm":0.00037627191460952637,"update-mean/_layer_42_gated_delta::out_proj":6.795142581950131e-08,"hidden-states-l1/_layer_49_attention::k_layernorm":3.394519256892116,"hidden-states-mean/_layer_53_gated_delta::core_attention":0.0009329591360117955,"moe/layer_17/act_rms_min":0.019387520849704742,"hidden-states-rms/_layer_38_gated_delta::out_norm":0.22100988154612583,"grad-rms/layer_39/moe::expert_fc1":2.0239286295356897e-08,"hidden-states-l1/_layer_37_gated_delta::core_attention":0.0004638161888518112,"param-mean/_layer_40_gated_delta::conv1d":0.0006856540567241609,"hidden-states-l1/_layer_09_moe::routed_experts":0.019494970646785997,"update-rms/_layer_14_gated_delta::conv1d":0.0008274219492719617,"grad-absmax/layer_27/gated_delta::conv1d":0.0002908060560002923,"hidden-states-absmax/_layer_16_input_layernorm":15,"moe/layer_14/act_rms_min_to_median":0.36659587021220374,"param-mean/_layer_29_gated_delta::out_norm":0.3919525146484375,"hidden-states-l1/_layer_40_residual_attn":2.502675116381063,"grad-absmax/layer_09/moe::shared_expert_fc2":5.204509307077387e-06} \ No newline at end of file